From 166ab40989529fe5e19b3534a07f07166da88e8f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 21 Mar 2025 13:27:12 -0700 Subject: [PATCH 01/31] chore(ci): run e2e tests on cmx --- e2e/cluster/cmx/cluster.go | 342 ++++++++++++++++++++ e2e/cluster/cmx/integration/cluster_test.go | 21 ++ e2e/cluster/cmx/network.go | 50 +++ e2e/cluster/cmx/node.go | 106 ++++++ e2e/cluster/lxd/utils.go | 8 - e2e/install_test.go | 29 +- 6 files changed, 538 insertions(+), 18 deletions(-) create mode 100644 e2e/cluster/cmx/cluster.go create mode 100644 e2e/cluster/cmx/integration/cluster_test.go create mode 100644 e2e/cluster/cmx/network.go create mode 100644 e2e/cluster/cmx/node.go diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go new file mode 100644 index 000000000..b980147e9 --- /dev/null +++ b/e2e/cluster/cmx/cluster.go @@ -0,0 +1,342 @@ +package cmx + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/google/uuid" +) + +const ( + DefaultDistribution = "ubuntu" + DefaultVersion = "22.04" + DefaultTTL = "2h" + DefaultInstanceType = "r1.medium" + DefaultDiskSize = 50 +) + +// Cluster implements the cluster.Cluster interface for Replicated VM +type Cluster struct { + t *testing.T + + gid string + networkID string + nodes []*node + proxyNode *node + sshUser string +} + +type ClusterInput struct { + T *testing.T + Nodes int + Distribution string + Version string + WithProxy bool + LicensePath string + EmbeddedClusterPath string + AirgapInstallBundlePath string + AirgapUpgradeBundlePath string +} + +// NewCluster creates a new CMX cluster using the provided configuration +func NewCluster(ctx context.Context, input ClusterInput) *Cluster { + if input.T == nil { + panic("testing.T is required") + } + + sshUser := os.Getenv("REPLICATEDVM_SSH_USER") + if sshUser == "" { + input.T.Fatalf("REPLICATEDVM_SSH_USER is not set") + } + + c := &Cluster{ + t: input.T, + sshUser: sshUser, + gid: uuid.New().String(), + } + c.t.Cleanup(c.destroy) + + c.t.Logf("Creating network") + network, err := createNetwork(ctx, c.gid, DefaultTTL) + if err != nil { + c.t.Fatalf("Failed to create network: %v", err) + } + c.networkID = network.ID + + c.t.Logf("Creating %d nodes", input.Nodes) + nodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) + if err != nil { + c.t.Fatalf("Failed to create nodes: %v", err) + } + c.nodes = nodes + + // If proxy is requested, create an additional node + if input.WithProxy { + c.t.Logf("Creating proxy node") + proxyNodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, createNodeOpts{ + Distribution: DefaultDistribution, + Version: DefaultVersion, + Count: 1, + InstanceType: "r1.small", + DiskSize: 10, + }) + if err != nil { + c.t.Fatalf("Failed to create proxy node: %v", err) + } + c.proxyNode = proxyNodes[0] + } + + for _, node := range c.nodes { + c.t.Logf("Copying files to node %s", node.ID) + err := c.copyFilesToNode(ctx, node, input) + if err != nil { + c.t.Fatalf("Failed to copy files to node %s: %v", node.ID, err) + } + c.t.Logf("Copying dirs to node %s", node.ID) + err = c.copyDirsToNode(ctx, node) + if err != nil { + c.t.Fatalf("Failed to copy dirs to node %s: %v", node.ID, err) + } + } + + return c +} + +func clusterInputToCreateNodeOpts(input ClusterInput) createNodeOpts { + opts := createNodeOpts{ + Distribution: input.Distribution, + Version: input.Version, + Count: input.Nodes, + } + if opts.Distribution == "" { + opts.Distribution = DefaultDistribution + } + if opts.Version == "" { + opts.Version = DefaultVersion + } + if opts.Count <= 0 { + opts.Count = 1 + } + if opts.InstanceType == "" { + opts.InstanceType = DefaultInstanceType + } + if opts.DiskSize <= 0 { + opts.DiskSize = DefaultDiskSize + } + return opts +} + +// Cleanup removes the VM instance +func (c *Cluster) Cleanup(envs ...map[string]string) { + // TODO: generate support bundle and copy playwright report + c.destroy() +} + +func (c *Cluster) destroy() { + if c.gid != "" { + // Best effort cleanup + c.t.Logf("Cleaning up nodes") + err := deleteNodesByGroupID(context.Background(), c.gid) + if err != nil { + c.t.Logf("Failed to cleanup cluster: %v", err) + } + } + + if c.networkID != "" { + c.t.Logf("Cleaning up network %s", c.networkID) + err := deleteNetwork(context.Background(), c.networkID) + if err != nil { + c.t.Logf("Failed to cleanup network: %v", err) + } + } +} + +// RunCommandOnNode executes a command on the specified node using replicated vm ssh +func (c *Cluster) RunCommandOnNode(ctx context.Context, node int, command []string, envs ...map[string]string) (string, string, error) { + c.t.Logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) + return c.runCommandOnNode(ctx, c.nodes[node], command, envs...) +} + +// RunCommandOnProxyNode executes a command on the proxy node +func (c *Cluster) RunCommandOnProxyNode(ctx context.Context, command []string, envs ...map[string]string) (string, string, error) { + c.t.Logf("Running command on proxy node: %s", strings.Join(command, " ")) + return c.runCommandOnNode(ctx, c.proxyNode, command, envs...) +} + +func (c *Cluster) Node(node int) *node { + return c.nodes[node] +} + +func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, command []string, envs ...map[string]string) (string, string, error) { + args := []string{} + args = append(args, sshConnectionArgs(node)...) + args = append(args, "sh", "-c", strings.Join(command, " ")) + cmd := exec.CommandContext(ctx, "ssh", args...) + + env := os.Environ() + for _, e := range envs { + for k, v := range e { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + } + cmd.Env = env + + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + + err := cmd.Run() + + stdout := outBuf.String() + stderr := errBuf.String() + + return stdout, stderr, err +} + +func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInput) error { + files := map[string]string{ + in.LicensePath: "/assets/license.yaml", //0644 + in.EmbeddedClusterPath: "/usr/local/bin/embedded-cluster", //0755 + in.AirgapInstallBundlePath: "/assets/ec-release.tgz", //0755 + in.AirgapUpgradeBundlePath: "/assets/ec-release-upgrade.tgz", //0755 + } + for src, dest := range files { + if src != "" { + err := c.CopyFileToNode(ctx, node, src, dest) + if err != nil { + return fmt.Errorf("copy file %s to node %s at %s: %v", src, node.ID, dest, err) + } + } + } + return nil +} + +func (c *Cluster) copyDirsToNode(ctx context.Context, node *node) error { + dirs := map[string]string{ + "../../../scripts": "/usr/local/bin", + "playwright": "/automation/playwright", + "../operator/charts/embedded-cluster-operator/troubleshoot": "/automation/troubleshoot", + } + for src, dest := range dirs { + err := c.CopyDirToNode(ctx, node, src, dest) + if err != nil { + return fmt.Errorf("copy dir %s to node %s at %s: %v", src, node.ID, dest, err) + } + } + return nil +} + +func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest string) error { + c.t.Logf("Copying file %s to node %s at %s", src, node.ID, dest) + + _, err := os.Stat(src) + if err != nil { + return fmt.Errorf("stat %s: %v", src, err) + } + + err = c.mkdirOnNode(ctx, node, filepath.Dir(dest)) + if err != nil { + return fmt.Errorf("mkdir %s on node %s: %v", filepath.Dir(dest), node.ID, err) + } + + args := []string{src} + args = append(args, sshConnectionArgs(node)...) + args[0] = fmt.Sprintf("%s:%s", args[0], dest) + scpCmd := exec.CommandContext(ctx, "scp", args...) + output, err := scpCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("err: %v, output: %s", err, string(output)) + } + return nil +} + +func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest string) error { + c.t.Logf("Copying dir %s to node %s at %s", src, node.ID, dest) + + _, err := os.Stat(src) + if err != nil { + return fmt.Errorf("stat %s: %v", src, err) + } + + err = c.mkdirOnNode(ctx, node, filepath.Dir(dest)) + if err != nil { + return fmt.Errorf("mkdir %s on node %s: %v", filepath.Dir(dest), node.ID, err) + } + + args := []string{src} + args = append(args, sshConnectionArgs(node)...) + args[0] = fmt.Sprintf("%s:%s", args[0], dest) + scpCmd := exec.CommandContext(ctx, "scp", args...) + output, err := scpCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("err: %v, output: %s", err, string(output)) + } + return nil +} + +func (c *Cluster) mkdirOnNode(ctx context.Context, node *node, dir string) error { + _, stderr, err := c.runCommandOnNode(ctx, node, []string{"mkdir", "-p", dir}, nil) + if err != nil { + return fmt.Errorf("err: %v, stderr: %s", err, stderr) + } + return nil +} + +func sshConnectionArgs(node *node) []string { + if sshUser := os.Getenv("REPLICATEDVM_SSH_USER"); sshUser != "" { + // If ssh user is provided, we can make a direct ssh connection + return []string{fmt.Sprintf("%s@%s", sshUser, node.DirectSSHEndpoint), "-p", strconv.Itoa(node.DirectSSHPort), "-o", "StrictHostKeyChecking=no"} + } + + sshDomain := os.Getenv("REPLICATEDVM_SSH_DOMAIN") + if sshDomain == "" { + sshDomain = "replicatedcluster.com" + } + return []string{fmt.Sprintf("%s@%s", node.ID, sshDomain), "-o", "StrictHostKeyChecking=no"} +} + +// SetupPlaywright installs necessary dependencies for Playwright testing +func (c *Cluster) SetupPlaywright(ctx context.Context, envs ...map[string]string) error { + c.t.Logf("Setting up Playwright") + + // Install Node.js and other dependencies + setupCommands := [][]string{ + {"curl", "-fsSL", "https://deb.nodesource.com/setup_16.x", "|", "sudo", "-E", "bash", "-"}, + {"sudo", "apt-get", "install", "-y", "nodejs"}, + {"npm", "install", "-g", "playwright"}, + {"npx", "playwright", "install"}, + } + + for _, cmd := range setupCommands { + _, stderr, err := c.RunCommandOnNode(ctx, 0, cmd, envs...) + if err != nil { + return fmt.Errorf("run command %q on node %s: %v, stderr: %s", strings.Join(cmd, " "), c.nodes[0].ID, err, stderr) + } + } + + return nil +} + +// SetupPlaywrightAndRunTest combines setup and test execution +func (c *Cluster) SetupPlaywrightAndRunTest(ctx context.Context, testName string, args ...string) (string, string, error) { + if err := c.SetupPlaywright(ctx); err != nil { + return "", "", err + } + return c.RunPlaywrightTest(ctx, testName, args...) +} + +// RunPlaywrightTest executes a Playwright test +func (c *Cluster) RunPlaywrightTest(ctx context.Context, testName string, args ...string) (string, string, error) { + c.t.Logf("Running Playwright test %s", testName) + + // Construct the test command + testCmd := append([]string{"npx", "playwright", "test", testName}, args...) + return c.RunCommandOnNode(ctx, 0, testCmd) +} diff --git a/e2e/cluster/cmx/integration/cluster_test.go b/e2e/cluster/cmx/integration/cluster_test.go new file mode 100644 index 000000000..2e149e428 --- /dev/null +++ b/e2e/cluster/cmx/integration/cluster_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "testing" + + "github.com/replicatedhq/embedded-cluster/e2e/cluster/cmx" +) + +func TestNewCluster(t *testing.T) { + _ = cmx.NewCluster(context.Background(), cmx.ClusterInput{ + T: t, + Nodes: 5, + Distribution: "ubuntu", + Version: "22.04", + WithProxy: true, + // AirgapInstallBundlePath: "/tmp/airgap-install-bundle.tar.gz", + // AirgapUpgradeBundlePath: "/tmp/airgap-upgrade-bundle.tar.gz", + }) + // defer cluster.Cleanup() +} diff --git a/e2e/cluster/cmx/network.go b/e2e/cluster/cmx/network.go new file mode 100644 index 000000000..8f103d61e --- /dev/null +++ b/e2e/cluster/cmx/network.go @@ -0,0 +1,50 @@ +package cmx + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" +) + +type network struct { + // ID is the unique identifier for the Network + ID string `json:"id"` + + // Status is the status of the Network + Status string `json:"status"` +} + +func createNetwork(ctx context.Context, gid string, ttl string) (*network, error) { + cmd := exec.CommandContext(ctx, "replicated", "network", "create", "--name", gid, "--ttl", ttl, "--wait", "2m", "--output", "json") + + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("err: %v, stderr: %s", err, errBuf.String()) + } + + var networks []*network + if err := json.Unmarshal(outBuf.Bytes(), &networks); err != nil { + return nil, fmt.Errorf("parse replicated network create output: %v", err) + } else if len(networks) == 0 { + return nil, fmt.Errorf("no network created") + } else if networks[0].Status != "running" { + return nil, fmt.Errorf("network %s is not running", networks[0].ID) + } + + return networks[0], nil +} + +func deleteNetwork(ctx context.Context, networkID string) error { + cmd := exec.CommandContext(ctx, "replicated", "network", "rm", "--name", networkID) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("err: %v, output: %s", err, string(output)) + } + return nil +} diff --git a/e2e/cluster/cmx/node.go b/e2e/cluster/cmx/node.go new file mode 100644 index 000000000..2ef1f139c --- /dev/null +++ b/e2e/cluster/cmx/node.go @@ -0,0 +1,106 @@ +package cmx + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "strconv" +) + +// createNodeOpts represents the configuration options for creating a VM node +type createNodeOpts struct { + // Distribution is the Linux distribution of the VM to provision + Distribution string + + // Version is the version to provision (format depends on distribution) + Version string + + // Count is the number of matching VMs to create (default 1) + Count int + + // InstanceType is the type of instance to use (e.g. r1.medium) + InstanceType string + + // DiskSize is the disk size in GiB to request per node (default 50) + DiskSize int +} + +// node represents a VM node instance +type node struct { + // ID is the unique identifier for the VM + ID string `json:"id"` + + // Status is the status of the VM + Status string `json:"status"` + + // DirectSSHEndpoint is the endpoint to connect to the VM via SSH + DirectSSHEndpoint string `json:"direct_ssh_endpoint"` + + // DirectSSHPort is the port to connect to the VM via SSH + DirectSSHPort int `json:"direct_ssh_port"` +} + +// createNodes creates nodes in the network with the given group ID and configuration +func createNodes(ctx context.Context, gid string, networkID string, ttl string, opts createNodeOpts) ([]*node, error) { + // Build the command with required flags + args := []string{ + "vm", "create", "-o", "json", + "--network", networkID, + "--tag", fmt.Sprintf("ec.e2e.group-id=%s", gid), + "--ttl", ttl, + "--wait", "5m", + } + + if opts.Distribution != "" { + args = append(args, "--distribution", opts.Distribution) + } + if opts.Version != "" { + args = append(args, "--version", opts.Version) + } + if opts.Count > 0 { + args = append(args, "--count", strconv.Itoa(opts.Count)) + } + if opts.InstanceType != "" { + args = append(args, "--instance-type", opts.InstanceType) + } + if opts.DiskSize > 0 { + args = append(args, "--disk", strconv.Itoa(opts.DiskSize)) + } + + // Execute replicated CLI command + cmd := exec.CommandContext(ctx, "replicated", args...) + + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("err: %v, stderr: %s", err, errBuf.String()) + } + + // Parse the JSON output + var nodes []*node + if err := json.Unmarshal(outBuf.Bytes(), &nodes); err != nil { + return nil, fmt.Errorf("parse replicated vm create output: %v", err) + } + + for _, node := range nodes { + if node.Status != "running" { + return nil, fmt.Errorf("VM %s is not running", node.ID) + } + } + + return nodes, nil +} + +func deleteNodesByGroupID(ctx context.Context, gid string) error { + cmd := exec.CommandContext(ctx, "replicated", "vm", "delete", "--tag", fmt.Sprintf("ec.e2e.group-id=%s", gid)) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("err: %v, output: %s", err, string(output)) + } + return nil +} diff --git a/e2e/cluster/lxd/utils.go b/e2e/cluster/lxd/utils.go index 193bf8e5d..addcb6f48 100644 --- a/e2e/cluster/lxd/utils.go +++ b/e2e/cluster/lxd/utils.go @@ -2,7 +2,6 @@ package lxd import ( "bytes" - "path/filepath" "strings" ) @@ -24,13 +23,6 @@ func mergeMaps(maps ...map[string]string) map[string]string { return merged } -func WithECShellEnv(dataDir string) map[string]string { - return map[string]string{ - "KUBECONFIG": filepath.Join(dataDir, "k0s/pki/admin.conf"), - "PATH": filepath.Join(dataDir, "bin"), - } -} - func WithMITMProxyEnv(nodeIPs []string) map[string]string { return map[string]string{ "HTTP_PROXY": HTTPMITMProxy, diff --git a/e2e/install_test.go b/e2e/install_test.go index 93cec87c5..5df46afde 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -1,10 +1,12 @@ package e2e import ( + "context" "encoding/base64" "encoding/json" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -13,6 +15,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "github.com/replicatedhq/embedded-cluster/e2e/cluster/cmx" "github.com/replicatedhq/embedded-cluster/e2e/cluster/docker" "github.com/replicatedhq/embedded-cluster/e2e/cluster/lxd" "github.com/replicatedhq/embedded-cluster/pkg/certs" @@ -2260,14 +2263,17 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) { func TestInstallWithPrivateCAs(t *testing.T) { RequireEnvVars(t, []string{"SHORT_SHA"}) - input := &lxd.ClusterInput{ + ctx := context.Background() + + input := cmx.ClusterInput{ T: t, Nodes: 1, - Image: "ubuntu/jammy", + Distribution: cmx.DefaultDistribution, + Version: cmx.DefaultVersion, LicensePath: "license.yaml", EmbeddedClusterPath: "../output/bin/embedded-cluster", } - tc := lxd.NewCluster(input) + tc := cmx.NewCluster(ctx, input) defer tc.Cleanup() certBuilder, err := certs.NewBuilder() @@ -2283,17 +2289,13 @@ func TestInstallWithPrivateCAs(t *testing.T) { require.NoError(t, err, "unable to write to temp file") tmpfile.Close() - lxd.CopyFileToNode(input, tc.Nodes[0], lxd.File{ - SourcePath: tmpfile.Name(), - DestPath: "/tmp/ca.crt", - Mode: 0666, - }) + tc.CopyFileToNode(ctx, tc.Node(0), tmpfile.Name(), "/tmp/ca.crt") installSingleNodeWithOptions(t, tc, installOptions{ privateCA: "/tmp/ca.crt", }) - if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { + if _, _, err := tc.SetupPlaywrightAndRunTest(ctx, "deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v", err) } @@ -2301,7 +2303,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { t.Logf("checking if the configmap was created with the right values") line := []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} - stdout, _, err := tc.RunCommandOnNode(0, line, lxd.WithECShellEnv("/var/lib/embedded-cluster")) + stdout, _, err := tc.RunCommandOnNode(ctx, 0, line, withECShellEnv("/var/lib/embedded-cluster")) require.NoError(t, err, "unable get kotsadm-private-cas configmap") var cm corev1.ConfigMap @@ -2518,3 +2520,10 @@ spec: t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } + +func withECShellEnv(dataDir string) map[string]string { + return map[string]string{ + "KUBECONFIG": filepath.Join(dataDir, "k0s/pki/admin.conf"), + "PATH": filepath.Join(dataDir, "bin"), + } +} From 4ace9a6857c29bb85b5482d7f9be13328e9e3ff4 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 21 Mar 2025 13:36:16 -0700 Subject: [PATCH 02/31] run e2e tests on cmx --- .github/actions/e2e/action.yml | 23 ++ .github/workflows/ci.yaml | 113 ++++---- Makefile | 3 + e2e/README.md | 98 ++++--- e2e/cluster/cmx/cluster.go | 256 +++++++++++++----- e2e/cluster/cmx/integration/cluster_test.go | 21 -- e2e/cluster/cmx/network.go | 15 +- e2e/cluster/cmx/node.go | 15 +- e2e/cluster/cmx/util.go | 48 ++++ e2e/install_test.go | 32 ++- e2e/scripts/airgap-update.sh | 2 +- e2e/scripts/airgap-update2.sh | 2 +- e2e/scripts/bypass-kurl-proxy.sh | 2 +- .../check-airgap-installation-state.sh | 2 +- e2e/scripts/check-airgap-post-ha-state.sh | 2 +- e2e/scripts/check-cidr-ranges.sh | 2 +- e2e/scripts/check-config-values.sh | 2 +- e2e/scripts/check-installation-state.sh | 2 +- e2e/scripts/check-nodes-removed.sh | 2 +- e2e/scripts/check-post-ha-state.sh | 2 +- e2e/scripts/check-post-restore.sh | 2 +- e2e/scripts/check-postupgrade-state.sh | 2 +- ...ck-pre-minio-removal-installation-state.sh | 2 +- e2e/scripts/check-velero-state.sh | 2 +- e2e/scripts/collect-support-bundle-cluster.sh | 2 +- .../collect-support-bundle-host-in-cluster.sh | 2 +- e2e/scripts/collect-support-bundle-host.sh | 2 +- e2e/scripts/embedded-preflight.sh | 2 +- e2e/scripts/install-kots-cli.sh | 2 +- e2e/scripts/kots-upstream-upgrade.sh | 2 +- e2e/scripts/pre-minio-removal-install.sh | 2 +- e2e/scripts/reset-installation.sh | 2 +- e2e/scripts/single-node-airgap-install.sh | 2 +- .../single-node-host-preflight-install.sh | 2 +- e2e/scripts/single-node-install.sh | 4 +- e2e/scripts/unsupported-overrides.sh | 2 +- e2e/scripts/vandoor-prepare.sh | 2 +- e2e/scripts/wait-for-ready-nodes.sh | 2 +- 38 files changed, 455 insertions(+), 227 deletions(-) delete mode 100644 e2e/cluster/cmx/integration/cluster_test.go create mode 100644 e2e/cluster/cmx/util.go diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index b70547e5f..4a7712cf7 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -43,6 +43,15 @@ inputs: version-specifier: description: 'the git sha or tag used to generate application version strings' required: true + cmx-replicated-api-token: + description: 'the replicated api token to use for cmx e2e tests' + required: false + replicatedvm-ssh-user: + description: 'the user to use for ssh to the replicated vm' + required: false + replicatedvm-ssh-key: + description: 'the ssh key to use for ssh to the replicated vm' + required: false upgrade-target-ec-version: description: 'the embedded cluster version to expect after upgrades complete' required: false # this is only set by post-release testing @@ -91,6 +100,20 @@ runs: with: go-version-file: go.mod cache-dependency-path: "**/*.sum" + - name: Install Replicated CLI + shell: bash + run: | + curl -o install.sh -sSL https://raw.githubusercontent.com/replicatedhq/replicated/master/install.sh + sudo bash ./install.sh + - name: Output CMX environment variables + shell: bash + if: ${{ inputs.cmx-replicated-api-token != '' && inputs.replicatedvm-ssh-key != '' && inputs.replicatedvm-ssh-user != '' }} + run: | + echo "${{ inputs.replicatedvm-ssh-key }}" | base64 --decode > /tmp/replicatedvm-ssh-key + chmod 400 /tmp/replicatedvm-ssh-key + echo "CMX_REPLICATED_API_TOKEN=${{ inputs.cmx-replicated-api-token }}" >> $GITHUB_ENV + echo "REPLICATEDVM_SSH_USER=${{ inputs.replicatedvm-ssh-user }}" >> $GITHUB_ENV + echo "REPLICATEDVM_SSH_KEY=/tmp/replicatedvm-ssh-key" >> $GITHUB_ENV - name: E2E shell: bash run: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 238722490..2543b1e6e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -632,36 +632,36 @@ jobs: matrix: test: - TestPreflights - - TestPreflightsNoexec - - TestMaterialize - - TestHostPreflightCustomSpec - - TestHostPreflightInBuiltSpec - - TestSingleNodeInstallation - - TestSingleNodeInstallationAlmaLinux8 - - TestSingleNodeInstallationDebian11 - - TestSingleNodeInstallationDebian12 - - TestSingleNodeInstallationCentos9Stream - - TestSingleNodeUpgradePreviousStable - - TestInstallFromReplicatedApp - - TestUpgradeFromReplicatedApp - - TestUpgradeEC18FromReplicatedApp - - TestResetAndReinstall - # - TestOldVersionUpgrade - - TestInstallSnapshotFromReplicatedApp - - TestMultiNodeInstallation - - TestMultiNodeHAInstallation - - TestSingleNodeDisasterRecovery - - TestSingleNodeLegacyDisasterRecovery - - TestSingleNodeResumeDisasterRecovery - - TestMultiNodeHADisasterRecovery - - TestSingleNodeInstallationNoopUpgrade - - TestCustomCIDR - - TestLocalArtifactMirror - - TestMultiNodeReset - - TestCollectSupportBundle - - TestUnsupportedOverrides - - TestHostCollectSupportBundleInCluster - - TestInstallWithConfigValues + # - TestPreflightsNoexec + # - TestMaterialize + # - TestHostPreflightCustomSpec + # - TestHostPreflightInBuiltSpec + # - TestSingleNodeInstallation + # - TestSingleNodeInstallationAlmaLinux8 + # - TestSingleNodeInstallationDebian11 + # - TestSingleNodeInstallationDebian12 + # - TestSingleNodeInstallationCentos9Stream + # - TestSingleNodeUpgradePreviousStable + # - TestInstallFromReplicatedApp + # - TestUpgradeFromReplicatedApp + # - TestUpgradeEC18FromReplicatedApp + # - TestResetAndReinstall + # # - TestOldVersionUpgrade + # - TestInstallSnapshotFromReplicatedApp + # - TestMultiNodeInstallation + # - TestMultiNodeHAInstallation + # - TestSingleNodeDisasterRecovery + # - TestSingleNodeLegacyDisasterRecovery + # - TestSingleNodeResumeDisasterRecovery + # - TestMultiNodeHADisasterRecovery + # - TestSingleNodeInstallationNoopUpgrade + # - TestCustomCIDR + # - TestLocalArtifactMirror + # - TestMultiNodeReset + # - TestCollectSupportBundle + # - TestUnsupportedOverrides + # - TestHostCollectSupportBundleInCluster + # - TestInstallWithConfigValues steps: - name: Checkout uses: actions/checkout@v4 @@ -725,32 +725,32 @@ jobs: fail-fast: false matrix: test: - - TestVersion - - TestCommandsRequireSudo - - TestResetAndReinstallAirgap - - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeConfigValues - - TestSingleNodeAirgapUpgradeCustomCIDR - - TestSingleNodeDisasterRecoveryWithProxy - - TestProxiedEnvironment - - TestProxiedCustomCIDR + # - TestVersion + # - TestCommandsRequireSudo + # - TestResetAndReinstallAirgap + # - TestSingleNodeAirgapUpgrade + # - TestSingleNodeAirgapUpgradeConfigValues + # - TestSingleNodeAirgapUpgradeCustomCIDR + # - TestSingleNodeDisasterRecoveryWithProxy + # - TestProxiedEnvironment + # - TestProxiedCustomCIDR - TestInstallWithPrivateCAs - - TestInstallWithMITMProxy - include: - - test: TestMultiNodeAirgapUpgrade - runner: embedded-cluster-2 - - test: TestMultiNodeAirgapUpgradeSameK0s - runner: embedded-cluster-2 - - test: TestMultiNodeAirgapUpgradePreviousStable - runner: embedded-cluster-2 - - test: TestAirgapUpgradeFromEC18 - runner: embedded-cluster-2 - - test: TestSingleNodeAirgapDisasterRecovery - runner: embedded-cluster-2 - - test: TestMultiNodeAirgapHAInstallation - runner: embedded-cluster-2 - - test: TestMultiNodeAirgapHADisasterRecovery - runner: embedded-cluster-2 + # - TestInstallWithMITMProxy + include: [] + # - test: TestMultiNodeAirgapUpgrade + # runner: embedded-cluster-2 + # - test: TestMultiNodeAirgapUpgradeSameK0s + # runner: embedded-cluster-2 + # - test: TestMultiNodeAirgapUpgradePreviousStable + # runner: embedded-cluster-2 + # - test: TestAirgapUpgradeFromEC18 + # runner: embedded-cluster-2 + # - test: TestSingleNodeAirgapDisasterRecovery + # runner: embedded-cluster-2 + # - test: TestMultiNodeAirgapHAInstallation + # runner: embedded-cluster-2 + # - test: TestMultiNodeAirgapHADisasterRecovery + # runner: embedded-cluster-2 steps: - name: Checkout uses: actions/checkout@v4 @@ -776,6 +776,9 @@ jobs: k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }} k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }} version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }} + cmx-replicated-api-token: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + replicatedvm-ssh-user: ${{ secrets.REPLICATEDVM_SSH_USER }} + replicatedvm-ssh-key: ${{ secrets.REPLICATEDVM_SSH_KEY }} e2e-main: name: E2E (on merge) diff --git a/Makefile b/Makefile index 0b6711912..af4b4af0d 100644 --- a/Makefile +++ b/Makefile @@ -284,9 +284,12 @@ vet: .PHONY: e2e-tests e2e-tests: embedded-release +e2e-tests: export SHORT_SHA = dev-$(shell git rev-parse --short HEAD) +e2e-tests: go test -tags $(GO_BUILD_TAGS) -timeout 60m -ldflags="$(LD_FLAGS)" -parallel 1 -failfast -v ./e2e .PHONY: e2e-test +e2e-test: export SHORT_SHA = dev-$(shell git rev-parse --short HEAD) e2e-test: go test -tags $(GO_BUILD_TAGS) -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$ diff --git a/e2e/README.md b/e2e/README.md index 157908601..259c052ef 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,4 +1,58 @@ -## E2E tests +## E2E Tests + +### Buildting the release for E2E tests + +```bash +./scripts/build-and-release.sh ARCH=amd64 APP_VERSION="appver-dev-local-$USER" +``` + +### Running individual CMX tests locally + +You can run a single test with: + +```bash +export SHORT_SHA="dev-local-$USER" +export LICENSE_ID="$EC_SMOKE_TEST_LICENSE_ID" +make e2e-test TEST_NAME=TestSomething +``` + +TestSomething is the name of the test function you want to run. + +### Running individual Docker tests locally + +You can run a single test with: + +```bash +export SHORT_SHA="dev-local-$USER" +export LICENSE_ID="$EC_SMOKE_TEST_LICENSE_ID" +make e2e-test TEST_NAME=TestSomething +``` + +TestSomething is the name of the test function you want to run. + +### Adding more tests + +Tests are added as Go tests in the e2e/ directory. +Tests must be added to the ci.yaml and release-prod.yaml GitHub workflows to be run in CI. + +### Kots test application + +During end to end tests we embed a license for a smoke test kots app. +This app can be found under the 'Replicated, Inc.' team on staging: + +https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app + +New releases are created using the corresponding YAML files in the e2e/kots-release-* directories. + +### Playwright + +We use [Playwright](https://playwright.dev/) to run end to end tests on the UI. +The tests live in the `playwright` directory. + +For more details on how to write tests with Playwright, refer to the [Playwright documentation](https://playwright.dev/docs/writing-tests). + + +## DEPRECATED: E2E tests Integration tests depends on LXD. The following procedure shows how to get everything installed on Ubuntu 22.04. @@ -31,45 +85,3 @@ Scripts inside the `scripts` directory are copied to all nodes. If you have a new test you want to add then start by creating a shell script to execute it and save it under the `scripts` dir. You can then call the script from your Go code. - -### Running all the tests - -You can run the tests from within this directory: - -``` -$ make e2e-tests -``` - -### Running individual tests - -You can run a single test with: - -``` -$ make e2e-test TEST_NAME=TestSomething -``` - -TestSomething is the name of the test function you want to run. - -### Adding more tests - -To add more tests you just need to create one inside this directory -and then add it to the `.github/workflows/e2e.yaml` file. - - -### Kots test application - -During end to end tests we embed a license for a smoke test kots app, -this app can be found under the 'Replicated, Inc.' team on staging: - -https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app - -Make sure to update the application yaml files under kots-release-onmerge -and kots-release-onpr directories if you create a new release of the remote -application. - -### Playwright - -We use [Playwright](https://playwright.dev/) to run end to end tests on the UI. -The tests live in the `playwright` directory. - -For more details on how to write tests with Playwright, refer to the [Playwright documentation](https://playwright.dev/docs/writing-tests). diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index b980147e9..1510c2c61 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -4,14 +4,17 @@ import ( "bytes" "context" "fmt" + "net" "os" "os/exec" "path/filepath" "strconv" "strings" "testing" + "time" "github.com/google/uuid" + "golang.org/x/sync/errgroup" ) const ( @@ -30,7 +33,6 @@ type Cluster struct { networkID string nodes []*node proxyNode *node - sshUser string } type ClusterInput struct { @@ -51,15 +53,16 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { panic("testing.T is required") } - sshUser := os.Getenv("REPLICATEDVM_SSH_USER") - if sshUser == "" { + if val := os.Getenv("REPLICATEDVM_SSH_USER"); val == "" { input.T.Fatalf("REPLICATEDVM_SSH_USER is not set") } + if val := os.Getenv("CMX_REPLICATED_API_TOKEN"); val == "" { + input.T.Fatalf("CMX_REPLICATED_API_TOKEN is not set") + } c := &Cluster{ - t: input.T, - sshUser: sshUser, - gid: uuid.New().String(), + t: input.T, + gid: uuid.New().String(), } c.t.Cleanup(c.destroy) @@ -70,16 +73,23 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { } c.networkID = network.ID - c.t.Logf("Creating %d nodes", input.Nodes) - nodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) - if err != nil { - c.t.Fatalf("Failed to create nodes: %v", err) - } - c.nodes = nodes + eg := errgroup.Group{} - // If proxy is requested, create an additional node - if input.WithProxy { + eg.Go(func() error { + c.t.Logf("Creating %d node(s)", input.Nodes) + start := time.Now() + nodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) + if err != nil { + return fmt.Errorf("create nodes: %v", err) + } + c.nodes = nodes + c.t.Logf("-> Created %d nodes in %s", len(nodes), time.Since(start)) + return nil + }) + + eg.Go(func() error { c.t.Logf("Creating proxy node") + start := time.Now() proxyNodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, createNodeOpts{ Distribution: DefaultDistribution, Version: DefaultVersion, @@ -88,22 +98,74 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { DiskSize: 10, }) if err != nil { - c.t.Fatalf("Failed to create proxy node: %v", err) + return fmt.Errorf("create proxy node: %v", err) } c.proxyNode = proxyNodes[0] - } - for _, node := range c.nodes { - c.t.Logf("Copying files to node %s", node.ID) - err := c.copyFilesToNode(ctx, node, input) + c.t.Logf("Enabling SSH access on proxy node") + err = c.enableSSHAccessOnNode(ctx, c.proxyNode) if err != nil { - c.t.Fatalf("Failed to copy files to node %s: %v", node.ID, err) + return fmt.Errorf("enable ssh access on proxy node: %v", err) } - c.t.Logf("Copying dirs to node %s", node.ID) - err = c.copyDirsToNode(ctx, node) + + c.t.Logf("Copying dirs to proxy node") + err = c.copyDirsToNode(ctx, c.proxyNode) if err != nil { - c.t.Fatalf("Failed to copy dirs to node %s: %v", node.ID, err) + return fmt.Errorf("copy dirs to proxy node: %v", err) } + + c.t.Logf("-> Created proxy node in %s", time.Since(start)) + return nil + }) + + err = eg.Wait() + if err != nil { + c.t.Fatalf("Failed to create nodes: %v", err) + } + + if input.WithProxy { + c.t.Logf("Configuring proxy") + // TODO: ConfigureProxy + } + + eg = errgroup.Group{} + + for _, node := range c.nodes { + eg.Go(func() error { + start := time.Now() + + c.t.Logf("Enabling SSH access on node %s", node.ID) + err := c.enableSSHAccessOnNode(ctx, node) + if err != nil { + return fmt.Errorf("enable ssh access: %v", err) + } + + c.t.Logf("Copying files to node %s", node.ID) + err = c.copyFilesToNode(ctx, node, input) + if err != nil { + return fmt.Errorf("copy files to node %s: %v", node.ID, err) + } + + c.t.Logf("Copying dirs to node %s", node.ID) + err = c.copyDirsToNode(ctx, node) + if err != nil { + return fmt.Errorf("copy dirs to node %s: %v", node.ID, err) + } + + c.t.Logf("Installing dependencies on node %s", node.ID) + _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"/automation/scripts/install-deps.sh"}) + if err != nil { + return fmt.Errorf("install dependencies on node %s: %v, stderr: %s", node.ID, err, stderr) + } + + c.t.Logf("-> Initialized node %s in %s", node.ID, time.Since(start)) + return nil + }) + } + + err = eg.Wait() + if err != nil { + c.t.Fatalf("Failed to copy files and dirs to nodes: %v", err) } return c @@ -160,24 +222,48 @@ func (c *Cluster) destroy() { // RunCommandOnNode executes a command on the specified node using replicated vm ssh func (c *Cluster) RunCommandOnNode(ctx context.Context, node int, command []string, envs ...map[string]string) (string, string, error) { + start := time.Now() c.t.Logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) - return c.runCommandOnNode(ctx, c.nodes[node], command, envs...) + stdout, stderr, err := c.runCommandOnNode(ctx, c.nodes[node], "root", command, envs...) + if err != nil { + return stdout, stderr, err + } + c.t.Logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) + return "", "", nil } // RunCommandOnProxyNode executes a command on the proxy node func (c *Cluster) RunCommandOnProxyNode(ctx context.Context, command []string, envs ...map[string]string) (string, string, error) { + start := time.Now() c.t.Logf("Running command on proxy node: %s", strings.Join(command, " ")) - return c.runCommandOnNode(ctx, c.proxyNode, command, envs...) + stdout, stderr, err := c.runCommandOnNode(ctx, c.proxyNode, "root", command, envs...) + if err != nil { + return stdout, stderr, err + } + c.t.Logf(" -> Command on proxy node completed in %s", time.Since(start)) + return stdout, stderr, nil +} + +func (c *Cluster) RunRegularUserCommandOnNode(ctx context.Context, node int, command []string, envs ...map[string]string) (string, string, error) { + start := time.Now() + c.t.Logf("Running command on node %s as user %s: %s", c.nodes[node].ID, os.Getenv("REPLICATEDVM_SSH_USER"), strings.Join(command, " ")) + stdout, stderr, err := c.runCommandOnNode(ctx, c.nodes[node], os.Getenv("REPLICATEDVM_SSH_USER"), command, envs...) + if err != nil { + return stdout, stderr, err + } + c.t.Logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) + return stdout, stderr, nil } func (c *Cluster) Node(node int) *node { return c.nodes[node] } -func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, command []string, envs ...map[string]string) (string, string, error) { +func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, sshUser string, command []string, envs ...map[string]string) (string, string, error) { args := []string{} - args = append(args, sshConnectionArgs(node)...) - args = append(args, "sh", "-c", strings.Join(command, " ")) + args = append(args, sshConnectionArgs(node, sshUser, false)...) + args = append(args, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) + c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) cmd := exec.CommandContext(ctx, "ssh", args...) env := os.Environ() @@ -200,6 +286,19 @@ func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, command []st return stdout, stderr, err } +func (c *Cluster) enableSSHAccessOnNode(ctx context.Context, node *node) error { + c.t.Logf("Enabling SSH access with root user on node %s", node.ID) + command := []string{ + "sudo", "mkdir", "-p", "/root/.ssh", + "&&", "sudo", "cp", "-f", "$HOME/.ssh/authorized_keys", "/root/.ssh/authorized_keys", + } + _, stderr, err := c.runCommandOnNode(ctx, node, os.Getenv("REPLICATEDVM_SSH_USER"), command) + if err != nil { + return fmt.Errorf("enable SSH access with root user: %v, stderr: %s", err, stderr) + } + return nil +} + func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInput) error { files := map[string]string{ in.LicensePath: "/assets/license.yaml", //0644 @@ -220,8 +319,8 @@ func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInp func (c *Cluster) copyDirsToNode(ctx context.Context, node *node) error { dirs := map[string]string{ - "../../../scripts": "/usr/local/bin", - "playwright": "/automation/playwright", + "scripts": "/automation/scripts", + "playwright": "/automation/playwright", "../operator/charts/embedded-cluster-operator/troubleshoot": "/automation/troubleshoot", } for src, dest := range dirs { @@ -234,6 +333,7 @@ func (c *Cluster) copyDirsToNode(ctx context.Context, node *node) error { } func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest string) error { + start := time.Now() c.t.Logf("Copying file %s to node %s at %s", src, node.ID, dest) _, err := os.Stat(src) @@ -246,18 +346,24 @@ func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest stri return fmt.Errorf("mkdir %s on node %s: %v", filepath.Dir(dest), node.ID, err) } - args := []string{src} - args = append(args, sshConnectionArgs(node)...) - args[0] = fmt.Sprintf("%s:%s", args[0], dest) + args := []string{} + args = append(args, sshConnectionArgs(node, "root", true)...) + args[len(args)-1] = fmt.Sprintf("%s:%s", args[len(args)-1], dest) + args = append(args[0:len(args)-1], "-p", src, args[len(args)-1]) + + c.t.Logf(" -> Running scp command on node %s: %q", node.ID, args) scpCmd := exec.CommandContext(ctx, "scp", args...) output, err := scpCmd.CombinedOutput() if err != nil { return fmt.Errorf("err: %v, output: %s", err, string(output)) } + + c.t.Logf(" -> Copied file %s to node %s in %s", src, node.ID, time.Since(start)) return nil } func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest string) error { + start := time.Now() c.t.Logf("Copying dir %s to node %s at %s", src, node.ID, dest) _, err := os.Stat(src) @@ -265,62 +371,71 @@ func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest strin return fmt.Errorf("stat %s: %v", src, err) } - err = c.mkdirOnNode(ctx, node, filepath.Dir(dest)) + srcTar, err := tmpFileName("*.tar.gz") if err != nil { - return fmt.Errorf("mkdir %s on node %s: %v", filepath.Dir(dest), node.ID, err) + return fmt.Errorf("create temp file: %v", err) } - args := []string{src} - args = append(args, sshConnectionArgs(node)...) - args[0] = fmt.Sprintf("%s:%s", args[0], dest) - scpCmd := exec.CommandContext(ctx, "scp", args...) - output, err := scpCmd.CombinedOutput() + err = tgzDir(ctx, src, srcTar) if err != nil { - return fmt.Errorf("err: %v, output: %s", err, string(output)) + return fmt.Errorf("tgz dir %s: %v", src, err) + } + defer os.Remove(srcTar) + + archiveDst := filepath.Join(filepath.Dir(dest), srcTar) + err = c.CopyFileToNode(ctx, node, srcTar, archiveDst) + if err != nil { + return fmt.Errorf("copy file %s to node %s at %s: %v", srcTar, node.ID, archiveDst, err) } + + envs := map[string]string{ + "COPYFILE_DISABLE": "true", // disable metadata files on macOS + } + _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"tar", "-xzf", archiveDst, "-C", filepath.Dir(dest)}, envs) + if err != nil { + return fmt.Errorf("run command: %v, stderr: %s", err, stderr) + } + + c.t.Logf(" -> Copied dir %s to node %s in %s", src, node.ID, time.Since(start)) return nil } func (c *Cluster) mkdirOnNode(ctx context.Context, node *node, dir string) error { - _, stderr, err := c.runCommandOnNode(ctx, node, []string{"mkdir", "-p", dir}, nil) + _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"mkdir", "-p", dir}, nil) if err != nil { return fmt.Errorf("err: %v, stderr: %s", err, stderr) } return nil } -func sshConnectionArgs(node *node) []string { - if sshUser := os.Getenv("REPLICATEDVM_SSH_USER"); sshUser != "" { - // If ssh user is provided, we can make a direct ssh connection - return []string{fmt.Sprintf("%s@%s", sshUser, node.DirectSSHEndpoint), "-p", strconv.Itoa(node.DirectSSHPort), "-o", "StrictHostKeyChecking=no"} - } +func sshConnectionArgs(node *node, sshUser string, isSCP bool) []string { + args := []string{"-o", "StrictHostKeyChecking=no"} - sshDomain := os.Getenv("REPLICATEDVM_SSH_DOMAIN") - if sshDomain == "" { - sshDomain = "replicatedcluster.com" + // If ssh user is provided, we can make a direct ssh connection + if sshKey := os.Getenv("REPLICATEDVM_SSH_KEY"); sshKey != "" { + args = append(args, "-i", sshKey) + } + if isSCP { + args = append(args, "-P", strconv.Itoa(node.DirectSSHPort)) + } else { + args = append(args, "-p", strconv.Itoa(node.DirectSSHPort)) } - return []string{fmt.Sprintf("%s@%s", node.ID, sshDomain), "-o", "StrictHostKeyChecking=no"} + args = append(args, fmt.Sprintf("%s@%s", sshUser, node.DirectSSHEndpoint)) + return args } // SetupPlaywright installs necessary dependencies for Playwright testing func (c *Cluster) SetupPlaywright(ctx context.Context, envs ...map[string]string) error { c.t.Logf("Setting up Playwright") - // Install Node.js and other dependencies - setupCommands := [][]string{ - {"curl", "-fsSL", "https://deb.nodesource.com/setup_16.x", "|", "sudo", "-E", "bash", "-"}, - {"sudo", "apt-get", "install", "-y", "nodejs"}, - {"npm", "install", "-g", "playwright"}, - {"npx", "playwright", "install"}, + line := []string{"/automation/scripts/bypass-kurl-proxy.sh"} + if _, stderr, err := c.RunCommandOnNode(ctx, 0, line, envs...); err != nil { + return fmt.Errorf("bypass kurl-proxy on proxy node: %v: %s", err, string(stderr)) } - - for _, cmd := range setupCommands { - _, stderr, err := c.RunCommandOnNode(ctx, 0, cmd, envs...) - if err != nil { - return fmt.Errorf("run command %q on node %s: %v, stderr: %s", strings.Join(cmd, " "), c.nodes[0].ID, err, stderr) - } + line = []string{"/automation/scripts/install-playwright.sh"} + if _, stderr, err := c.RunCommandOnProxyNode(ctx, line); err != nil { + return fmt.Errorf("install playwright on proxy node: %v: %s", err, string(stderr)) } - return nil } @@ -336,7 +451,14 @@ func (c *Cluster) SetupPlaywrightAndRunTest(ctx context.Context, testName string func (c *Cluster) RunPlaywrightTest(ctx context.Context, testName string, args ...string) (string, string, error) { c.t.Logf("Running Playwright test %s", testName) - // Construct the test command - testCmd := append([]string{"npx", "playwright", "test", testName}, args...) - return c.RunCommandOnNode(ctx, 0, testCmd) + line := []string{"/automation/scripts/playwright.sh", testName} + line = append(line, args...) + env := map[string]string{ + "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("TODO", "30003")), // TODO: get ip and expose port + } + stdout, stderr, err := c.RunCommandOnProxyNode(ctx, line, env) + if err != nil { + return stdout, stderr, fmt.Errorf("run playwright test %s on proxy node: %v", testName, err) + } + return stdout, stderr, nil } diff --git a/e2e/cluster/cmx/integration/cluster_test.go b/e2e/cluster/cmx/integration/cluster_test.go deleted file mode 100644 index 2e149e428..000000000 --- a/e2e/cluster/cmx/integration/cluster_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "context" - "testing" - - "github.com/replicatedhq/embedded-cluster/e2e/cluster/cmx" -) - -func TestNewCluster(t *testing.T) { - _ = cmx.NewCluster(context.Background(), cmx.ClusterInput{ - T: t, - Nodes: 5, - Distribution: "ubuntu", - Version: "22.04", - WithProxy: true, - // AirgapInstallBundlePath: "/tmp/airgap-install-bundle.tar.gz", - // AirgapUpgradeBundlePath: "/tmp/airgap-upgrade-bundle.tar.gz", - }) - // defer cluster.Cleanup() -} diff --git a/e2e/cluster/cmx/network.go b/e2e/cluster/cmx/network.go index 8f103d61e..ef7b65101 100644 --- a/e2e/cluster/cmx/network.go +++ b/e2e/cluster/cmx/network.go @@ -19,11 +19,17 @@ type network struct { func createNetwork(ctx context.Context, gid string, ttl string) (*network, error) { cmd := exec.CommandContext(ctx, "replicated", "network", "create", "--name", gid, "--ttl", ttl, "--wait", "2m", "--output", "json") + apiTokenEnv, err := replicatedApiTokenEnv() + if err != nil { + return nil, err + } + cmd.Env = append(cmd.Environ(), apiTokenEnv...) + var outBuf, errBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &errBuf - err := cmd.Run() + err = cmd.Run() if err != nil { return nil, fmt.Errorf("err: %v, stderr: %s", err, errBuf.String()) } @@ -42,6 +48,13 @@ func createNetwork(ctx context.Context, gid string, ttl string) (*network, error func deleteNetwork(ctx context.Context, networkID string) error { cmd := exec.CommandContext(ctx, "replicated", "network", "rm", "--name", networkID) + + apiTokenEnv, err := replicatedApiTokenEnv() + if err != nil { + return err + } + cmd.Env = append(cmd.Environ(), apiTokenEnv...) + output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("err: %v, output: %s", err, string(output)) diff --git a/e2e/cluster/cmx/node.go b/e2e/cluster/cmx/node.go index 2ef1f139c..05f626556 100644 --- a/e2e/cluster/cmx/node.go +++ b/e2e/cluster/cmx/node.go @@ -72,11 +72,17 @@ func createNodes(ctx context.Context, gid string, networkID string, ttl string, // Execute replicated CLI command cmd := exec.CommandContext(ctx, "replicated", args...) + apiTokenEnv, err := replicatedApiTokenEnv() + if err != nil { + return nil, err + } + cmd.Env = append(cmd.Environ(), apiTokenEnv...) + var outBuf, errBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &errBuf - err := cmd.Run() + err = cmd.Run() if err != nil { return nil, fmt.Errorf("err: %v, stderr: %s", err, errBuf.String()) } @@ -98,6 +104,13 @@ func createNodes(ctx context.Context, gid string, networkID string, ttl string, func deleteNodesByGroupID(ctx context.Context, gid string) error { cmd := exec.CommandContext(ctx, "replicated", "vm", "delete", "--tag", fmt.Sprintf("ec.e2e.group-id=%s", gid)) + + apiTokenEnv, err := replicatedApiTokenEnv() + if err != nil { + return err + } + cmd.Env = append(cmd.Environ(), apiTokenEnv...) + output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("err: %v, output: %s", err, string(output)) diff --git a/e2e/cluster/cmx/util.go b/e2e/cluster/cmx/util.go new file mode 100644 index 000000000..bef02eb68 --- /dev/null +++ b/e2e/cluster/cmx/util.go @@ -0,0 +1,48 @@ +package cmx + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" +) + +func replicatedApiTokenEnv() ([]string, error) { + if val := os.Getenv("CMX_REPLICATED_API_TOKEN"); val != "" { + return []string{fmt.Sprintf("REPLICATED_API_TOKEN=%s", val), "REPLICATED_API_ORIGIN=", "REPLICATED_APP="}, nil + } + return nil, fmt.Errorf("CMX_REPLICATED_API_TOKEN is not set") +} + +func tmpFileName(pattern string) (string, error) { + srcTar, err := os.CreateTemp("", pattern) + if err != nil { + return "", fmt.Errorf("create temp file: %v", err) + } + name := srcTar.Name() + err = srcTar.Close() + if err != nil { + return "", fmt.Errorf("close temp file: %v", err) + } + err = os.Remove(name) + if err != nil { + return "", fmt.Errorf("remove temp file: %v", err) + } + return name, nil +} + +func tgzDir(ctx context.Context, src string, dst string) error { + cmd := exec.CommandContext(ctx, "tar", "-czf", dst, filepath.Base(src)) + cmd.Dir = filepath.Dir(src) + + var errBuf bytes.Buffer + cmd.Stderr = &errBuf + + err := cmd.Run() + if err != nil { + return fmt.Errorf("failed to create tar archive: %v, stderr: %s", err, errBuf.String()) + } + return nil +} diff --git a/e2e/install_test.go b/e2e/install_test.go index 5df46afde..97eacb47a 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2266,12 +2266,10 @@ func TestInstallWithPrivateCAs(t *testing.T) { ctx := context.Background() input := cmx.ClusterInput{ - T: t, - Nodes: 1, - Distribution: cmx.DefaultDistribution, - Version: cmx.DefaultVersion, - LicensePath: "license.yaml", - EmbeddedClusterPath: "../output/bin/embedded-cluster", + T: t, + Nodes: 1, + Distribution: cmx.DefaultDistribution, + Version: cmx.DefaultVersion, } tc := cmx.NewCluster(ctx, input) defer tc.Cleanup() @@ -2291,18 +2289,32 @@ func TestInstallWithPrivateCAs(t *testing.T) { tc.CopyFileToNode(ctx, tc.Node(0), tmpfile.Name(), "/tmp/ca.crt") + t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line := []string{"/automation/scripts/vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} + if stdout, stderr, err := tc.RunCommandOnNode(ctx, 0, line); err != nil { + t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) + } + installSingleNodeWithOptions(t, tc, installOptions{ - privateCA: "/tmp/ca.crt", + licensePath: "license.yaml", + privateCA: "/tmp/ca.crt", }) - if _, _, err := tc.SetupPlaywrightAndRunTest(ctx, "deploy-app"); err != nil { - t.Fatalf("fail to run playwright test deploy-app: %v", err) + t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) + line = []string{"/automation/scripts/single-node-install.sh", "ui", os.Getenv("SHORT_SHA"), "--private-ca", "/tmp/ca.crt"} + if stdout, stderr, err := tc.RunCommandOnNode(ctx, 0, line); err != nil { + t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: deploying app", time.Now().Format(time.RFC3339)) + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest(ctx, "deploy-app"); err != nil { + t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } checkInstallationState(t, tc) t.Logf("checking if the configmap was created with the right values") - line := []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} + line = []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} stdout, _, err := tc.RunCommandOnNode(ctx, 0, line, withECShellEnv("/var/lib/embedded-cluster")) require.NoError(t, err, "unable get kotsadm-private-cas configmap") diff --git a/e2e/scripts/airgap-update.sh b/e2e/scripts/airgap-update.sh index c8b834492..48cb9f659 100755 --- a/e2e/scripts/airgap-update.sh +++ b/e2e/scripts/airgap-update.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/airgap-update2.sh b/e2e/scripts/airgap-update2.sh index 2d16b935b..dbfce47a9 100755 --- a/e2e/scripts/airgap-update2.sh +++ b/e2e/scripts/airgap-update2.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/bypass-kurl-proxy.sh b/e2e/scripts/bypass-kurl-proxy.sh index e8a4e1e7b..e0abf9e68 100755 --- a/e2e/scripts/bypass-kurl-proxy.sh +++ b/e2e/scripts/bypass-kurl-proxy.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-airgap-installation-state.sh b/e2e/scripts/check-airgap-installation-state.sh index 86eb98855..6e830d2d5 100755 --- a/e2e/scripts/check-airgap-installation-state.sh +++ b/e2e/scripts/check-airgap-installation-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-airgap-post-ha-state.sh b/e2e/scripts/check-airgap-post-ha-state.sh index 95043e0a0..329deab27 100755 --- a/e2e/scripts/check-airgap-post-ha-state.sh +++ b/e2e/scripts/check-airgap-post-ha-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-cidr-ranges.sh b/e2e/scripts/check-cidr-ranges.sh index 60ee34bff..3d3baeaff 100755 --- a/e2e/scripts/check-cidr-ranges.sh +++ b/e2e/scripts/check-cidr-ranges.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-config-values.sh b/e2e/scripts/check-config-values.sh index 064af70c4..cba3c2d59 100755 --- a/e2e/scripts/check-config-values.sh +++ b/e2e/scripts/check-config-values.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index 1937b1711..9f47fa00f 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-nodes-removed.sh b/e2e/scripts/check-nodes-removed.sh index 9938d80c4..6607bbfb7 100755 --- a/e2e/scripts/check-nodes-removed.sh +++ b/e2e/scripts/check-nodes-removed.sh @@ -3,7 +3,7 @@ # It fails if the cluster size isn't exactly what we expect set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-post-ha-state.sh b/e2e/scripts/check-post-ha-state.sh index 29c8b81c2..3a7dd4bcf 100755 --- a/e2e/scripts/check-post-ha-state.sh +++ b/e2e/scripts/check-post-ha-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-post-restore.sh b/e2e/scripts/check-post-restore.sh index c5f11982a..724eadb08 100755 --- a/e2e/scripts/check-post-restore.sh +++ b/e2e/scripts/check-post-restore.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-postupgrade-state.sh b/e2e/scripts/check-postupgrade-state.sh index bbb7adba0..0e393afda 100755 --- a/e2e/scripts/check-postupgrade-state.sh +++ b/e2e/scripts/check-postupgrade-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh function check_nginx_version { diff --git a/e2e/scripts/check-pre-minio-removal-installation-state.sh b/e2e/scripts/check-pre-minio-removal-installation-state.sh index 7b0a93717..2c0599507 100755 --- a/e2e/scripts/check-pre-minio-removal-installation-state.sh +++ b/e2e/scripts/check-pre-minio-removal-installation-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/check-velero-state.sh b/e2e/scripts/check-velero-state.sh index 34f8d33c2..7ef7debca 100755 --- a/e2e/scripts/check-velero-state.sh +++ b/e2e/scripts/check-velero-state.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh wait_for_velero_pods() { diff --git a/e2e/scripts/collect-support-bundle-cluster.sh b/e2e/scripts/collect-support-bundle-cluster.sh index 9c1aa05e6..3d267b445 100755 --- a/e2e/scripts/collect-support-bundle-cluster.sh +++ b/e2e/scripts/collect-support-bundle-cluster.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/collect-support-bundle-host-in-cluster.sh b/e2e/scripts/collect-support-bundle-host-in-cluster.sh index 6b4d73186..d4c085b59 100755 --- a/e2e/scripts/collect-support-bundle-host-in-cluster.sh +++ b/e2e/scripts/collect-support-bundle-host-in-cluster.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/collect-support-bundle-host.sh b/e2e/scripts/collect-support-bundle-host.sh index e41239415..ea82af1a3 100755 --- a/e2e/scripts/collect-support-bundle-host.sh +++ b/e2e/scripts/collect-support-bundle-host.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/embedded-preflight.sh b/e2e/scripts/embedded-preflight.sh index a7a92c490..8158c3b30 100755 --- a/e2e/scripts/embedded-preflight.sh +++ b/e2e/scripts/embedded-preflight.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh diff --git a/e2e/scripts/install-kots-cli.sh b/e2e/scripts/install-kots-cli.sh index b6fec3ff2..57e5a1c72 100755 --- a/e2e/scripts/install-kots-cli.sh +++ b/e2e/scripts/install-kots-cli.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh function main() { diff --git a/e2e/scripts/kots-upstream-upgrade.sh b/e2e/scripts/kots-upstream-upgrade.sh index 8e7eb2340..e5fa07217 100755 --- a/e2e/scripts/kots-upstream-upgrade.sh +++ b/e2e/scripts/kots-upstream-upgrade.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/pre-minio-removal-install.sh b/e2e/scripts/pre-minio-removal-install.sh index c38f92a24..64af68a13 100755 --- a/e2e/scripts/pre-minio-removal-install.sh +++ b/e2e/scripts/pre-minio-removal-install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh deploy_app() { diff --git a/e2e/scripts/reset-installation.sh b/e2e/scripts/reset-installation.sh index fbb298503..458562f65 100755 --- a/e2e/scripts/reset-installation.sh +++ b/e2e/scripts/reset-installation.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/single-node-airgap-install.sh b/e2e/scripts/single-node-airgap-install.sh index f7ab3581a..fd77c4e90 100755 --- a/e2e/scripts/single-node-airgap-install.sh +++ b/e2e/scripts/single-node-airgap-install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh check_openebs_storage_class() { diff --git a/e2e/scripts/single-node-host-preflight-install.sh b/e2e/scripts/single-node-host-preflight-install.sh index e8683b26d..0a1b72308 100755 --- a/e2e/scripts/single-node-host-preflight-install.sh +++ b/e2e/scripts/single-node-host-preflight-install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh index 22f948ff7..4a45f8555 100755 --- a/e2e/scripts/single-node-install.sh +++ b/e2e/scripts/single-node-install.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin -. $DIR/common.sh +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +. "$DIR/common.sh" deploy_app() { echo "getting apps" diff --git a/e2e/scripts/unsupported-overrides.sh b/e2e/scripts/unsupported-overrides.sh index d92cf9100..bacb679d9 100755 --- a/e2e/scripts/unsupported-overrides.sh +++ b/e2e/scripts/unsupported-overrides.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh override_applied() { diff --git a/e2e/scripts/vandoor-prepare.sh b/e2e/scripts/vandoor-prepare.sh index 55d6825ee..38b3b1114 100755 --- a/e2e/scripts/vandoor-prepare.sh +++ b/e2e/scripts/vandoor-prepare.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { diff --git a/e2e/scripts/wait-for-ready-nodes.sh b/e2e/scripts/wait-for-ready-nodes.sh index d59bcb87c..ef1c03243 100755 --- a/e2e/scripts/wait-for-ready-nodes.sh +++ b/e2e/scripts/wait-for-ready-nodes.sh @@ -2,7 +2,7 @@ # This script waits for X nodes to be ready. X is the first argument. set -euox pipefail -DIR=/usr/local/bin +DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . $DIR/common.sh main() { From eb034b6d7c750b6e80253703559239ae5890e429 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 10:23:09 -0700 Subject: [PATCH 03/31] f --- e2e/cluster/cmx/cluster.go | 207 +++++++++++++++++++------------------ e2e/cluster/interface.go | 2 + e2e/install_test.go | 13 ++- 3 files changed, 115 insertions(+), 107 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 1510c2c61..7c0acc038 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -67,7 +67,7 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { c.t.Cleanup(c.destroy) c.t.Logf("Creating network") - network, err := createNetwork(ctx, c.gid, DefaultTTL) + network, err := createNetwork(c.t.Context(), c.gid, DefaultTTL) if err != nil { c.t.Fatalf("Failed to create network: %v", err) } @@ -78,7 +78,7 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { eg.Go(func() error { c.t.Logf("Creating %d node(s)", input.Nodes) start := time.Now() - nodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) + nodes, err := createNodes(c.t.Context(), c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) if err != nil { return fmt.Errorf("create nodes: %v", err) } @@ -90,7 +90,7 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { eg.Go(func() error { c.t.Logf("Creating proxy node") start := time.Now() - proxyNodes, err := createNodes(ctx, c.gid, c.networkID, DefaultTTL, createNodeOpts{ + proxyNodes, err := createNodes(c.t.Context(), c.gid, c.networkID, DefaultTTL, createNodeOpts{ Distribution: DefaultDistribution, Version: DefaultVersion, Count: 1, @@ -103,13 +103,13 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { c.proxyNode = proxyNodes[0] c.t.Logf("Enabling SSH access on proxy node") - err = c.enableSSHAccessOnNode(ctx, c.proxyNode) + err = c.enableSSHAccessOnNode(c.proxyNode) if err != nil { return fmt.Errorf("enable ssh access on proxy node: %v", err) } c.t.Logf("Copying dirs to proxy node") - err = c.copyDirsToNode(ctx, c.proxyNode) + err = c.copyDirsToNode(c.proxyNode) if err != nil { return fmt.Errorf("copy dirs to proxy node: %v", err) } @@ -135,25 +135,25 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { start := time.Now() c.t.Logf("Enabling SSH access on node %s", node.ID) - err := c.enableSSHAccessOnNode(ctx, node) + err := c.enableSSHAccessOnNode(node) if err != nil { return fmt.Errorf("enable ssh access: %v", err) } c.t.Logf("Copying files to node %s", node.ID) - err = c.copyFilesToNode(ctx, node, input) + err = c.copyFilesToNode(node, input) if err != nil { return fmt.Errorf("copy files to node %s: %v", node.ID, err) } c.t.Logf("Copying dirs to node %s", node.ID) - err = c.copyDirsToNode(ctx, node) + err = c.copyDirsToNode(node) if err != nil { return fmt.Errorf("copy dirs to node %s: %v", node.ID, err) } c.t.Logf("Installing dependencies on node %s", node.ID) - _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"/automation/scripts/install-deps.sh"}) + _, stderr, err := c.runCommandOnNode(node, "root", []string{"/automation/scripts/install-deps.sh"}) if err != nil { return fmt.Errorf("install dependencies on node %s: %v, stderr: %s", node.ID, err, stderr) } @@ -195,36 +195,11 @@ func clusterInputToCreateNodeOpts(input ClusterInput) createNodeOpts { return opts } -// Cleanup removes the VM instance -func (c *Cluster) Cleanup(envs ...map[string]string) { - // TODO: generate support bundle and copy playwright report - c.destroy() -} - -func (c *Cluster) destroy() { - if c.gid != "" { - // Best effort cleanup - c.t.Logf("Cleaning up nodes") - err := deleteNodesByGroupID(context.Background(), c.gid) - if err != nil { - c.t.Logf("Failed to cleanup cluster: %v", err) - } - } - - if c.networkID != "" { - c.t.Logf("Cleaning up network %s", c.networkID) - err := deleteNetwork(context.Background(), c.networkID) - if err != nil { - c.t.Logf("Failed to cleanup network: %v", err) - } - } -} - -// RunCommandOnNode executes a command on the specified node using replicated vm ssh -func (c *Cluster) RunCommandOnNode(ctx context.Context, node int, command []string, envs ...map[string]string) (string, string, error) { +// RunCommandOnNode executes a command on the specified node as the root user +func (c *Cluster) RunCommandOnNode(node int, command []string, envs ...map[string]string) (string, string, error) { start := time.Now() c.t.Logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) - stdout, stderr, err := c.runCommandOnNode(ctx, c.nodes[node], "root", command, envs...) + stdout, stderr, err := c.runCommandOnNode(c.nodes[node], "root", command, envs...) if err != nil { return stdout, stderr, err } @@ -232,11 +207,11 @@ func (c *Cluster) RunCommandOnNode(ctx context.Context, node int, command []stri return "", "", nil } -// RunCommandOnProxyNode executes a command on the proxy node -func (c *Cluster) RunCommandOnProxyNode(ctx context.Context, command []string, envs ...map[string]string) (string, string, error) { +// RunCommandOnProxyNode executes a command on the proxy node as the root user +func (c *Cluster) RunCommandOnProxyNode(command []string, envs ...map[string]string) (string, string, error) { start := time.Now() c.t.Logf("Running command on proxy node: %s", strings.Join(command, " ")) - stdout, stderr, err := c.runCommandOnNode(ctx, c.proxyNode, "root", command, envs...) + stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", command, envs...) if err != nil { return stdout, stderr, err } @@ -244,10 +219,11 @@ func (c *Cluster) RunCommandOnProxyNode(ctx context.Context, command []string, e return stdout, stderr, nil } -func (c *Cluster) RunRegularUserCommandOnNode(ctx context.Context, node int, command []string, envs ...map[string]string) (string, string, error) { +// RunRegularUserCommandOnNode executes a command on the specified node as a non-root user +func (c *Cluster) RunRegularUserCommandOnNode(node int, command []string, envs ...map[string]string) (string, string, error) { start := time.Now() c.t.Logf("Running command on node %s as user %s: %s", c.nodes[node].ID, os.Getenv("REPLICATEDVM_SSH_USER"), strings.Join(command, " ")) - stdout, stderr, err := c.runCommandOnNode(ctx, c.nodes[node], os.Getenv("REPLICATEDVM_SSH_USER"), command, envs...) + stdout, stderr, err := c.runCommandOnNode(c.nodes[node], os.Getenv("REPLICATEDVM_SSH_USER"), command, envs...) if err != nil { return stdout, stderr, err } @@ -255,16 +231,86 @@ func (c *Cluster) RunRegularUserCommandOnNode(ctx context.Context, node int, com return stdout, stderr, nil } -func (c *Cluster) Node(node int) *node { - return c.nodes[node] +// Cleanup removes the VM instance +func (c *Cluster) Cleanup(envs ...map[string]string) { + // TODO: generate support bundle and copy playwright report + c.destroy() +} + +// CopyFileToNode copies a file to a node +func (c *Cluster) CopyFileToNode(node int, src, dest string) error { + return c.copyFileToNode(c.nodes[node], src, dest) +} + +// CopyDirToNode copies a directory to a node +func (c *Cluster) CopyDirToNode(node int, src, dest string) error { + return c.copyDirToNode(c.nodes[node], src, dest) +} + +// SetupPlaywright installs necessary dependencies for Playwright testing +func (c *Cluster) SetupPlaywright(envs ...map[string]string) error { + c.t.Logf("Setting up Playwright") + + line := []string{"/automation/scripts/bypass-kurl-proxy.sh"} + if _, stderr, err := c.RunCommandOnNode(0, line, envs...); err != nil { + return fmt.Errorf("bypass kurl-proxy on proxy node: %v: %s", err, string(stderr)) + } + line = []string{"/automation/scripts/install-playwright.sh"} + if _, stderr, err := c.RunCommandOnProxyNode(line); err != nil { + return fmt.Errorf("install playwright on proxy node: %v: %s", err, string(stderr)) + } + return nil +} + +// SetupPlaywrightAndRunTest combines setup and test execution +func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (string, string, error) { + if err := c.SetupPlaywright(); err != nil { + return "", "", err + } + return c.RunPlaywrightTest(testName, args...) +} + +// RunPlaywrightTest executes a Playwright test +func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, string, error) { + c.t.Logf("Running Playwright test %s", testName) + + line := []string{"/automation/scripts/playwright.sh", testName} + line = append(line, args...) + env := map[string]string{ + "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("TODO", "30003")), // TODO: get ip and expose port + } + stdout, stderr, err := c.RunCommandOnProxyNode(line, env) + if err != nil { + return stdout, stderr, fmt.Errorf("run playwright test %s on proxy node: %v", testName, err) + } + return stdout, stderr, nil +} + +func (c *Cluster) destroy() { + if c.gid != "" { + // Best effort cleanup + c.t.Logf("Cleaning up nodes") + err := deleteNodesByGroupID(context.Background(), c.gid) + if err != nil { + c.t.Logf("Failed to cleanup cluster: %v", err) + } + } + + if c.networkID != "" { + c.t.Logf("Cleaning up network %s", c.networkID) + err := deleteNetwork(context.Background(), c.networkID) + if err != nil { + c.t.Logf("Failed to cleanup network: %v", err) + } + } } -func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, sshUser string, command []string, envs ...map[string]string) (string, string, error) { +func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, envs ...map[string]string) (string, string, error) { args := []string{} args = append(args, sshConnectionArgs(node, sshUser, false)...) args = append(args, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) - cmd := exec.CommandContext(ctx, "ssh", args...) + cmd := exec.CommandContext(c.t.Context(), "ssh", args...) env := os.Environ() for _, e := range envs { @@ -286,20 +332,20 @@ func (c *Cluster) runCommandOnNode(ctx context.Context, node *node, sshUser stri return stdout, stderr, err } -func (c *Cluster) enableSSHAccessOnNode(ctx context.Context, node *node) error { +func (c *Cluster) enableSSHAccessOnNode(node *node) error { c.t.Logf("Enabling SSH access with root user on node %s", node.ID) command := []string{ "sudo", "mkdir", "-p", "/root/.ssh", "&&", "sudo", "cp", "-f", "$HOME/.ssh/authorized_keys", "/root/.ssh/authorized_keys", } - _, stderr, err := c.runCommandOnNode(ctx, node, os.Getenv("REPLICATEDVM_SSH_USER"), command) + _, stderr, err := c.runCommandOnNode(node, os.Getenv("REPLICATEDVM_SSH_USER"), command) if err != nil { return fmt.Errorf("enable SSH access with root user: %v, stderr: %s", err, stderr) } return nil } -func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInput) error { +func (c *Cluster) copyFilesToNode(node *node, in ClusterInput) error { files := map[string]string{ in.LicensePath: "/assets/license.yaml", //0644 in.EmbeddedClusterPath: "/usr/local/bin/embedded-cluster", //0755 @@ -308,7 +354,7 @@ func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInp } for src, dest := range files { if src != "" { - err := c.CopyFileToNode(ctx, node, src, dest) + err := c.copyFileToNode(node, src, dest) if err != nil { return fmt.Errorf("copy file %s to node %s at %s: %v", src, node.ID, dest, err) } @@ -317,14 +363,14 @@ func (c *Cluster) copyFilesToNode(ctx context.Context, node *node, in ClusterInp return nil } -func (c *Cluster) copyDirsToNode(ctx context.Context, node *node) error { +func (c *Cluster) copyDirsToNode(node *node) error { dirs := map[string]string{ "scripts": "/automation/scripts", "playwright": "/automation/playwright", "../operator/charts/embedded-cluster-operator/troubleshoot": "/automation/troubleshoot", } for src, dest := range dirs { - err := c.CopyDirToNode(ctx, node, src, dest) + err := c.copyDirToNode(node, src, dest) if err != nil { return fmt.Errorf("copy dir %s to node %s at %s: %v", src, node.ID, dest, err) } @@ -332,7 +378,7 @@ func (c *Cluster) copyDirsToNode(ctx context.Context, node *node) error { return nil } -func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest string) error { +func (c *Cluster) copyFileToNode(node *node, src, dest string) error { start := time.Now() c.t.Logf("Copying file %s to node %s at %s", src, node.ID, dest) @@ -341,7 +387,7 @@ func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest stri return fmt.Errorf("stat %s: %v", src, err) } - err = c.mkdirOnNode(ctx, node, filepath.Dir(dest)) + err = c.mkdirOnNode(node, filepath.Dir(dest)) if err != nil { return fmt.Errorf("mkdir %s on node %s: %v", filepath.Dir(dest), node.ID, err) } @@ -352,7 +398,7 @@ func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest stri args = append(args[0:len(args)-1], "-p", src, args[len(args)-1]) c.t.Logf(" -> Running scp command on node %s: %q", node.ID, args) - scpCmd := exec.CommandContext(ctx, "scp", args...) + scpCmd := exec.CommandContext(c.t.Context(), "scp", args...) output, err := scpCmd.CombinedOutput() if err != nil { return fmt.Errorf("err: %v, output: %s", err, string(output)) @@ -362,7 +408,7 @@ func (c *Cluster) CopyFileToNode(ctx context.Context, node *node, src, dest stri return nil } -func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest string) error { +func (c *Cluster) copyDirToNode(node *node, src, dest string) error { start := time.Now() c.t.Logf("Copying dir %s to node %s at %s", src, node.ID, dest) @@ -376,14 +422,14 @@ func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest strin return fmt.Errorf("create temp file: %v", err) } - err = tgzDir(ctx, src, srcTar) + err = tgzDir(c.t.Context(), src, srcTar) if err != nil { return fmt.Errorf("tgz dir %s: %v", src, err) } defer os.Remove(srcTar) archiveDst := filepath.Join(filepath.Dir(dest), srcTar) - err = c.CopyFileToNode(ctx, node, srcTar, archiveDst) + err = c.copyFileToNode(node, srcTar, archiveDst) if err != nil { return fmt.Errorf("copy file %s to node %s at %s: %v", srcTar, node.ID, archiveDst, err) } @@ -391,7 +437,7 @@ func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest strin envs := map[string]string{ "COPYFILE_DISABLE": "true", // disable metadata files on macOS } - _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"tar", "-xzf", archiveDst, "-C", filepath.Dir(dest)}, envs) + _, stderr, err := c.runCommandOnNode(node, "root", []string{"tar", "-xzf", archiveDst, "-C", filepath.Dir(dest)}, envs) if err != nil { return fmt.Errorf("run command: %v, stderr: %s", err, stderr) } @@ -400,8 +446,8 @@ func (c *Cluster) CopyDirToNode(ctx context.Context, node *node, src, dest strin return nil } -func (c *Cluster) mkdirOnNode(ctx context.Context, node *node, dir string) error { - _, stderr, err := c.runCommandOnNode(ctx, node, "root", []string{"mkdir", "-p", dir}, nil) +func (c *Cluster) mkdirOnNode(node *node, dir string) error { + _, stderr, err := c.runCommandOnNode(node, "root", []string{"mkdir", "-p", dir}, nil) if err != nil { return fmt.Errorf("err: %v, stderr: %s", err, stderr) } @@ -423,42 +469,3 @@ func sshConnectionArgs(node *node, sshUser string, isSCP bool) []string { args = append(args, fmt.Sprintf("%s@%s", sshUser, node.DirectSSHEndpoint)) return args } - -// SetupPlaywright installs necessary dependencies for Playwright testing -func (c *Cluster) SetupPlaywright(ctx context.Context, envs ...map[string]string) error { - c.t.Logf("Setting up Playwright") - - line := []string{"/automation/scripts/bypass-kurl-proxy.sh"} - if _, stderr, err := c.RunCommandOnNode(ctx, 0, line, envs...); err != nil { - return fmt.Errorf("bypass kurl-proxy on proxy node: %v: %s", err, string(stderr)) - } - line = []string{"/automation/scripts/install-playwright.sh"} - if _, stderr, err := c.RunCommandOnProxyNode(ctx, line); err != nil { - return fmt.Errorf("install playwright on proxy node: %v: %s", err, string(stderr)) - } - return nil -} - -// SetupPlaywrightAndRunTest combines setup and test execution -func (c *Cluster) SetupPlaywrightAndRunTest(ctx context.Context, testName string, args ...string) (string, string, error) { - if err := c.SetupPlaywright(ctx); err != nil { - return "", "", err - } - return c.RunPlaywrightTest(ctx, testName, args...) -} - -// RunPlaywrightTest executes a Playwright test -func (c *Cluster) RunPlaywrightTest(ctx context.Context, testName string, args ...string) (string, string, error) { - c.t.Logf("Running Playwright test %s", testName) - - line := []string{"/automation/scripts/playwright.sh", testName} - line = append(line, args...) - env := map[string]string{ - "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("TODO", "30003")), // TODO: get ip and expose port - } - stdout, stderr, err := c.RunCommandOnProxyNode(ctx, line, env) - if err != nil { - return stdout, stderr, fmt.Errorf("run playwright test %s on proxy node: %v", testName, err) - } - return stdout, stderr, nil -} diff --git a/e2e/cluster/interface.go b/e2e/cluster/interface.go index 420135510..1f883c989 100644 --- a/e2e/cluster/interface.go +++ b/e2e/cluster/interface.go @@ -1,6 +1,7 @@ package cluster import ( + "github.com/replicatedhq/embedded-cluster/e2e/cluster/cmx" "github.com/replicatedhq/embedded-cluster/e2e/cluster/docker" "github.com/replicatedhq/embedded-cluster/e2e/cluster/lxd" ) @@ -8,6 +9,7 @@ import ( var ( _ Cluster = (*lxd.Cluster)(nil) _ Cluster = (*docker.Cluster)(nil) + _ Cluster = (*cmx.Cluster)(nil) ) type Cluster interface { diff --git a/e2e/install_test.go b/e2e/install_test.go index 97eacb47a..421cc2c17 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2287,27 +2287,26 @@ func TestInstallWithPrivateCAs(t *testing.T) { require.NoError(t, err, "unable to write to temp file") tmpfile.Close() - tc.CopyFileToNode(ctx, tc.Node(0), tmpfile.Name(), "/tmp/ca.crt") + tc.CopyFileToNode(0, tmpfile.Name(), "/tmp/ca.crt") t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line := []string{"/automation/scripts/vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} - if stdout, stderr, err := tc.RunCommandOnNode(ctx, 0, line); err != nil { + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } installSingleNodeWithOptions(t, tc, installOptions{ - licensePath: "license.yaml", - privateCA: "/tmp/ca.crt", + privateCA: "/tmp/ca.crt", }) t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) line = []string{"/automation/scripts/single-node-install.sh", "ui", os.Getenv("SHORT_SHA"), "--private-ca", "/tmp/ca.crt"} - if stdout, stderr, err := tc.RunCommandOnNode(ctx, 0, line); err != nil { + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } t.Logf("%s: deploying app", time.Now().Format(time.RFC3339)) - if stdout, stderr, err := tc.SetupPlaywrightAndRunTest(ctx, "deploy-app"); err != nil { + if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) } @@ -2315,7 +2314,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { t.Logf("checking if the configmap was created with the right values") line = []string{"kubectl", "get", "cm", "kotsadm-private-cas", "-n", "kotsadm", "-o", "json"} - stdout, _, err := tc.RunCommandOnNode(ctx, 0, line, withECShellEnv("/var/lib/embedded-cluster")) + stdout, _, err := tc.RunCommandOnNode(0, line, withECShellEnv("/var/lib/embedded-cluster")) require.NoError(t, err, "unable get kotsadm-private-cas configmap") var cm corev1.ConfigMap From b9864bc32d6673ec7e1e45278f04e12d864cc44b Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 10:24:38 -0700 Subject: [PATCH 04/31] f --- e2e/install_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/e2e/install_test.go b/e2e/install_test.go index 421cc2c17..294a0f7ce 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2299,12 +2299,6 @@ func TestInstallWithPrivateCAs(t *testing.T) { privateCA: "/tmp/ca.crt", }) - t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) - line = []string{"/automation/scripts/single-node-install.sh", "ui", os.Getenv("SHORT_SHA"), "--private-ca", "/tmp/ca.crt"} - if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { - t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) - } - t.Logf("%s: deploying app", time.Now().Format(time.RFC3339)) if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) From 4c351e0625176f41d204847adeb066b69d535e5a Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 10:33:18 -0700 Subject: [PATCH 05/31] f --- e2e/cluster/cmx/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 7c0acc038..b0bc921fa 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -277,7 +277,7 @@ func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, st line := []string{"/automation/scripts/playwright.sh", testName} line = append(line, args...) env := map[string]string{ - "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("TODO", "30003")), // TODO: get ip and expose port + "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.1", "30003")), } stdout, stderr, err := c.RunCommandOnProxyNode(line, env) if err != nil { From 82878b049236f3403b37b1bc03023c6ab8082d36 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 10:58:31 -0700 Subject: [PATCH 06/31] f --- e2e/cluster/cmx/cluster.go | 12 ++++++++---- e2e/install_test.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index b0bc921fa..07bf9fc4d 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -153,7 +153,7 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { } c.t.Logf("Installing dependencies on node %s", node.ID) - _, stderr, err := c.runCommandOnNode(node, "root", []string{"/automation/scripts/install-deps.sh"}) + _, stderr, err := c.runCommandOnNode(node, "root", []string{"install-deps.sh"}) if err != nil { return fmt.Errorf("install dependencies on node %s: %v, stderr: %s", node.ID, err, stderr) } @@ -251,11 +251,11 @@ func (c *Cluster) CopyDirToNode(node int, src, dest string) error { func (c *Cluster) SetupPlaywright(envs ...map[string]string) error { c.t.Logf("Setting up Playwright") - line := []string{"/automation/scripts/bypass-kurl-proxy.sh"} + line := []string{"bypass-kurl-proxy.sh"} if _, stderr, err := c.RunCommandOnNode(0, line, envs...); err != nil { return fmt.Errorf("bypass kurl-proxy on proxy node: %v: %s", err, string(stderr)) } - line = []string{"/automation/scripts/install-playwright.sh"} + line = []string{"install-playwright.sh"} if _, stderr, err := c.RunCommandOnProxyNode(line); err != nil { return fmt.Errorf("install playwright on proxy node: %v: %s", err, string(stderr)) } @@ -274,7 +274,7 @@ func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (st func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, string, error) { c.t.Logf("Running Playwright test %s", testName) - line := []string{"/automation/scripts/playwright.sh", testName} + line := []string{"playwright.sh", testName} line = append(line, args...) env := map[string]string{ "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.1", "30003")), @@ -375,6 +375,10 @@ func (c *Cluster) copyDirsToNode(node *node) error { return fmt.Errorf("copy dir %s to node %s at %s: %v", src, node.ID, dest, err) } } + _, stderr, err := c.runCommandOnNode(node, "root", []string{"cp", "-r", "/automation/scripts/*", "/usr/local/bin"}) + if err != nil { + return fmt.Errorf("copy scripts to /usr/local/bin: %v, stderr: %s", err, stderr) + } return nil } diff --git a/e2e/install_test.go b/e2e/install_test.go index 294a0f7ce..78686f5b3 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2290,7 +2290,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { tc.CopyFileToNode(0, tmpfile.Name(), "/tmp/ca.crt") t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339)) - line := []string{"/automation/scripts/vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} + line := []string{"vandoor-prepare.sh", fmt.Sprintf("appver-%s", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } From 52d794cd980ea1d7f145072c77ea58a9fbf10701 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 11:19:04 -0700 Subject: [PATCH 07/31] f --- e2e/cluster/cmx/cluster.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 07bf9fc4d..f7958693f 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -364,10 +364,13 @@ func (c *Cluster) copyFilesToNode(node *node, in ClusterInput) error { } func (c *Cluster) copyDirsToNode(node *node) error { + // Get the directory of the current test file + testDir := filepath.Dir(c.t.Name()) + dirs := map[string]string{ - "scripts": "/automation/scripts", - "playwright": "/automation/playwright", - "../operator/charts/embedded-cluster-operator/troubleshoot": "/automation/troubleshoot", + filepath.Join(testDir, "scripts"): "/automation/scripts", + filepath.Join(testDir, "playwright"): "/automation/playwright", + filepath.Join(testDir, "../operator/charts/embedded-cluster-operator/troubleshoot"): "/automation/troubleshoot", } for src, dest := range dirs { err := c.copyDirToNode(node, src, dest) From def53664396753053d00e4d178717b3846554c59 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 11:29:19 -0700 Subject: [PATCH 08/31] f --- e2e/cluster/cmx/cluster.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index f7958693f..07bf9fc4d 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -364,13 +364,10 @@ func (c *Cluster) copyFilesToNode(node *node, in ClusterInput) error { } func (c *Cluster) copyDirsToNode(node *node) error { - // Get the directory of the current test file - testDir := filepath.Dir(c.t.Name()) - dirs := map[string]string{ - filepath.Join(testDir, "scripts"): "/automation/scripts", - filepath.Join(testDir, "playwright"): "/automation/playwright", - filepath.Join(testDir, "../operator/charts/embedded-cluster-operator/troubleshoot"): "/automation/troubleshoot", + "scripts": "/automation/scripts", + "playwright": "/automation/playwright", + "../operator/charts/embedded-cluster-operator/troubleshoot": "/automation/troubleshoot", } for src, dest := range dirs { err := c.copyDirToNode(node, src, dest) From 9f4192a1c69f7269d3f9e9dc64868cc4c5d3e81e Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 11:46:39 -0700 Subject: [PATCH 09/31] f --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index af4b4af0d..b3b568941 100644 --- a/Makefile +++ b/Makefile @@ -284,12 +284,10 @@ vet: .PHONY: e2e-tests e2e-tests: embedded-release -e2e-tests: export SHORT_SHA = dev-$(shell git rev-parse --short HEAD) e2e-tests: go test -tags $(GO_BUILD_TAGS) -timeout 60m -ldflags="$(LD_FLAGS)" -parallel 1 -failfast -v ./e2e .PHONY: e2e-test -e2e-test: export SHORT_SHA = dev-$(shell git rev-parse --short HEAD) e2e-test: go test -tags $(GO_BUILD_TAGS) -timeout 60m -ldflags="$(LD_FLAGS)" -v ./e2e -run ^$(TEST_NAME)$$ From 721ba20ecf262e337feed8b4baecefca7508747f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 11:47:07 -0700 Subject: [PATCH 10/31] f --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index b3b568941..0b6711912 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,6 @@ vet: .PHONY: e2e-tests e2e-tests: embedded-release -e2e-tests: go test -tags $(GO_BUILD_TAGS) -timeout 60m -ldflags="$(LD_FLAGS)" -parallel 1 -failfast -v ./e2e .PHONY: e2e-test From 69a1e84abd99db4d1254a894379e46f9c27ca990 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 13:08:46 -0700 Subject: [PATCH 11/31] f --- e2e/cluster/cmx/cluster.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 07bf9fc4d..095a80b32 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -308,17 +308,18 @@ func (c *Cluster) destroy() { func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, envs ...map[string]string) (string, string, error) { args := []string{} args = append(args, sshConnectionArgs(node, sshUser, false)...) - args = append(args, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) - c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) - cmd := exec.CommandContext(c.t.Context(), "ssh", args...) - env := os.Environ() + envArr := []string{} for _, e := range envs { for k, v := range e { - env = append(env, fmt.Sprintf("%s=%s", k, v)) + envArr = append(envArr, fmt.Sprintf("export %s=%s;", k, v)) } } - cmd.Env = env + + args = append(args, fmt.Sprintf("%s; sh -c '%s'", strings.Join(envArr, " "), strings.Join(command, " "))) + + c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) + cmd := exec.CommandContext(c.t.Context(), "ssh", args...) var outBuf, errBuf bytes.Buffer cmd.Stdout = &outBuf From 885d6e4ad493ebdbcf6a77f117c8f15b3261e15f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 13:32:26 -0700 Subject: [PATCH 12/31] f --- e2e/cluster/cmx/cluster.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 095a80b32..b64ff45a3 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -309,14 +309,14 @@ func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, args := []string{} args = append(args, sshConnectionArgs(node, sshUser, false)...) - envArr := []string{} + cmdArr := []string{} for _, e := range envs { for k, v := range e { - envArr = append(envArr, fmt.Sprintf("export %s=%s;", k, v)) + cmdArr = append(cmdArr, fmt.Sprintf("export %s=%s", k, v)) } } - - args = append(args, fmt.Sprintf("%s; sh -c '%s'", strings.Join(envArr, " "), strings.Join(command, " "))) + cmdArr = append(cmdArr, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) + args = append(args, strings.Join(cmdArr, "; ")) c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) cmd := exec.CommandContext(c.t.Context(), "ssh", args...) From 3db208c4121a89fc7c5cd80e9b66b92c2bbeb010 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 14:24:28 -0700 Subject: [PATCH 13/31] f --- e2e/cluster/cmx/cluster.go | 171 +++++++++++++++++++++++++++---------- 1 file changed, 128 insertions(+), 43 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index b64ff45a3..129590d68 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "testing" "time" @@ -29,10 +30,11 @@ const ( type Cluster struct { t *testing.T - gid string - networkID string - nodes []*node - proxyNode *node + gid string + networkID string + nodes []*node + proxyNode *node + supportBundleNodeIndex int } type ClusterInput struct { @@ -45,6 +47,7 @@ type ClusterInput struct { EmbeddedClusterPath string AirgapInstallBundlePath string AirgapUpgradeBundlePath string + SupportBundleNodeIndex int } // NewCluster creates a new CMX cluster using the provided configuration @@ -61,12 +64,13 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { } c := &Cluster{ - t: input.T, - gid: uuid.New().String(), + t: input.T, + gid: uuid.New().String(), + supportBundleNodeIndex: input.SupportBundleNodeIndex, } c.t.Cleanup(c.destroy) - c.t.Logf("Creating network") + c.logf("Creating network") network, err := createNetwork(c.t.Context(), c.gid, DefaultTTL) if err != nil { c.t.Fatalf("Failed to create network: %v", err) @@ -76,19 +80,19 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { eg := errgroup.Group{} eg.Go(func() error { - c.t.Logf("Creating %d node(s)", input.Nodes) + c.logf("Creating %d node(s)", input.Nodes) start := time.Now() nodes, err := createNodes(c.t.Context(), c.gid, c.networkID, DefaultTTL, clusterInputToCreateNodeOpts(input)) if err != nil { return fmt.Errorf("create nodes: %v", err) } c.nodes = nodes - c.t.Logf("-> Created %d nodes in %s", len(nodes), time.Since(start)) + c.logf("-> Created %d nodes in %s", len(nodes), time.Since(start)) return nil }) eg.Go(func() error { - c.t.Logf("Creating proxy node") + c.logf("Creating proxy node") start := time.Now() proxyNodes, err := createNodes(c.t.Context(), c.gid, c.networkID, DefaultTTL, createNodeOpts{ Distribution: DefaultDistribution, @@ -102,19 +106,19 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { } c.proxyNode = proxyNodes[0] - c.t.Logf("Enabling SSH access on proxy node") + c.logf("Enabling SSH access on proxy node") err = c.enableSSHAccessOnNode(c.proxyNode) if err != nil { return fmt.Errorf("enable ssh access on proxy node: %v", err) } - c.t.Logf("Copying dirs to proxy node") + c.logf("Copying dirs to proxy node") err = c.copyDirsToNode(c.proxyNode) if err != nil { return fmt.Errorf("copy dirs to proxy node: %v", err) } - c.t.Logf("-> Created proxy node in %s", time.Since(start)) + c.logf("-> Created proxy node in %s", time.Since(start)) return nil }) @@ -124,7 +128,7 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { } if input.WithProxy { - c.t.Logf("Configuring proxy") + c.logf("Configuring proxy") // TODO: ConfigureProxy } @@ -134,31 +138,31 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { eg.Go(func() error { start := time.Now() - c.t.Logf("Enabling SSH access on node %s", node.ID) + c.logf("Enabling SSH access on node %s", node.ID) err := c.enableSSHAccessOnNode(node) if err != nil { return fmt.Errorf("enable ssh access: %v", err) } - c.t.Logf("Copying files to node %s", node.ID) + c.logf("Copying files to node %s", node.ID) err = c.copyFilesToNode(node, input) if err != nil { return fmt.Errorf("copy files to node %s: %v", node.ID, err) } - c.t.Logf("Copying dirs to node %s", node.ID) + c.logf("Copying dirs to node %s", node.ID) err = c.copyDirsToNode(node) if err != nil { return fmt.Errorf("copy dirs to node %s: %v", node.ID, err) } - c.t.Logf("Installing dependencies on node %s", node.ID) + c.logf("Installing dependencies on node %s", node.ID) _, stderr, err := c.runCommandOnNode(node, "root", []string{"install-deps.sh"}) if err != nil { return fmt.Errorf("install dependencies on node %s: %v, stderr: %s", node.ID, err, stderr) } - c.t.Logf("-> Initialized node %s in %s", node.ID, time.Since(start)) + c.logf("-> Initialized node %s in %s", node.ID, time.Since(start)) return nil }) } @@ -198,43 +202,43 @@ func clusterInputToCreateNodeOpts(input ClusterInput) createNodeOpts { // RunCommandOnNode executes a command on the specified node as the root user func (c *Cluster) RunCommandOnNode(node int, command []string, envs ...map[string]string) (string, string, error) { start := time.Now() - c.t.Logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) + c.logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.nodes[node], "root", command, envs...) if err != nil { return stdout, stderr, err } - c.t.Logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) + c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) return "", "", nil } // RunCommandOnProxyNode executes a command on the proxy node as the root user func (c *Cluster) RunCommandOnProxyNode(command []string, envs ...map[string]string) (string, string, error) { start := time.Now() - c.t.Logf("Running command on proxy node: %s", strings.Join(command, " ")) + c.logf("Running command on proxy node: %s", strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", command, envs...) if err != nil { return stdout, stderr, err } - c.t.Logf(" -> Command on proxy node completed in %s", time.Since(start)) + c.logf(" -> Command on proxy node completed in %s", time.Since(start)) return stdout, stderr, nil } // RunRegularUserCommandOnNode executes a command on the specified node as a non-root user func (c *Cluster) RunRegularUserCommandOnNode(node int, command []string, envs ...map[string]string) (string, string, error) { start := time.Now() - c.t.Logf("Running command on node %s as user %s: %s", c.nodes[node].ID, os.Getenv("REPLICATEDVM_SSH_USER"), strings.Join(command, " ")) + c.logf("Running command on node %s as user %s: %s", c.nodes[node].ID, os.Getenv("REPLICATEDVM_SSH_USER"), strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.nodes[node], os.Getenv("REPLICATEDVM_SSH_USER"), command, envs...) if err != nil { return stdout, stderr, err } - c.t.Logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) + c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) return stdout, stderr, nil } // Cleanup removes the VM instance func (c *Cluster) Cleanup(envs ...map[string]string) { - // TODO: generate support bundle and copy playwright report - c.destroy() + c.generateSupportBundle(envs...) + c.copyPlaywrightReport() } // CopyFileToNode copies a file to a node @@ -249,14 +253,14 @@ func (c *Cluster) CopyDirToNode(node int, src, dest string) error { // SetupPlaywright installs necessary dependencies for Playwright testing func (c *Cluster) SetupPlaywright(envs ...map[string]string) error { - c.t.Logf("Setting up Playwright") + c.logf("Setting up Playwright") line := []string{"bypass-kurl-proxy.sh"} - if _, stderr, err := c.RunCommandOnNode(0, line, envs...); err != nil { + if _, stderr, err := c.runCommandOnNode(c.nodes[0], "root", line, envs...); err != nil { return fmt.Errorf("bypass kurl-proxy on proxy node: %v: %s", err, string(stderr)) } line = []string{"install-playwright.sh"} - if _, stderr, err := c.RunCommandOnProxyNode(line); err != nil { + if _, stderr, err := c.runCommandOnNode(c.proxyNode, "root", line, envs...); err != nil { return fmt.Errorf("install playwright on proxy node: %v: %s", err, string(stderr)) } return nil @@ -272,14 +276,14 @@ func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (st // RunPlaywrightTest executes a Playwright test func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, string, error) { - c.t.Logf("Running Playwright test %s", testName) + c.logf("Running Playwright test %s", testName) line := []string{"playwright.sh", testName} line = append(line, args...) env := map[string]string{ "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.1", "30003")), } - stdout, stderr, err := c.RunCommandOnProxyNode(line, env) + stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", line, env) if err != nil { return stdout, stderr, fmt.Errorf("run playwright test %s on proxy node: %v", testName, err) } @@ -289,18 +293,18 @@ func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, st func (c *Cluster) destroy() { if c.gid != "" { // Best effort cleanup - c.t.Logf("Cleaning up nodes") + c.logf("Cleaning up nodes") err := deleteNodesByGroupID(context.Background(), c.gid) if err != nil { - c.t.Logf("Failed to cleanup cluster: %v", err) + c.logf("Failed to cleanup cluster: %v", err) } } if c.networkID != "" { - c.t.Logf("Cleaning up network %s", c.networkID) + c.logf("Cleaning up network %s", c.networkID) err := deleteNetwork(context.Background(), c.networkID) if err != nil { - c.t.Logf("Failed to cleanup network: %v", err) + c.logf("Failed to cleanup network: %v", err) } } } @@ -318,7 +322,7 @@ func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, cmdArr = append(cmdArr, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) args = append(args, strings.Join(cmdArr, "; ")) - c.t.Logf(" -> Running ssh command on node %s: %q", node.ID, args) + c.logf(" -> Running ssh command on node %s: %q", node.ID, args) cmd := exec.CommandContext(c.t.Context(), "ssh", args...) var outBuf, errBuf bytes.Buffer @@ -334,7 +338,7 @@ func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, } func (c *Cluster) enableSSHAccessOnNode(node *node) error { - c.t.Logf("Enabling SSH access with root user on node %s", node.ID) + c.logf("Enabling SSH access with root user on node %s", node.ID) command := []string{ "sudo", "mkdir", "-p", "/root/.ssh", "&&", "sudo", "cp", "-f", "$HOME/.ssh/authorized_keys", "/root/.ssh/authorized_keys", @@ -385,7 +389,7 @@ func (c *Cluster) copyDirsToNode(node *node) error { func (c *Cluster) copyFileToNode(node *node, src, dest string) error { start := time.Now() - c.t.Logf("Copying file %s to node %s at %s", src, node.ID, dest) + c.logf("Copying file %s to node %s at %s", src, node.ID, dest) _, err := os.Stat(src) if err != nil { @@ -402,20 +406,43 @@ func (c *Cluster) copyFileToNode(node *node, src, dest string) error { args[len(args)-1] = fmt.Sprintf("%s:%s", args[len(args)-1], dest) args = append(args[0:len(args)-1], "-p", src, args[len(args)-1]) - c.t.Logf(" -> Running scp command on node %s: %q", node.ID, args) + c.logf(" -> Running scp command on node %s: %q", node.ID, args) scpCmd := exec.CommandContext(c.t.Context(), "scp", args...) output, err := scpCmd.CombinedOutput() if err != nil { return fmt.Errorf("err: %v, output: %s", err, string(output)) } - c.t.Logf(" -> Copied file %s to node %s in %s", src, node.ID, time.Since(start)) + c.logf(" -> Copied file %s to node %s in %s", src, node.ID, time.Since(start)) + return nil +} + +func (c *Cluster) copyFileFromNode(node *node, src, dest string) error { + start := time.Now() + c.logf("Copying file %s from node %s to %s", src, node.ID, dest) + + dir := filepath.Dir(dest) + _ = os.MkdirAll(dir, 0755) + + args := []string{} + args = append(args, sshConnectionArgs(node, "root", true)...) + args[len(args)-1] = fmt.Sprintf("%s:%s", args[len(args)-1], src) + args = append(args[0:len(args)-1], "-p", args[len(args)-1], dest) + + c.logf(" -> Running scp command on node %s: %q", node.ID, args) + scpCmd := exec.CommandContext(c.t.Context(), "scp", args...) + output, err := scpCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("err: %v, output: %s", err, string(output)) + } + + c.logf(" -> Copied file %s from node %s in %s", src, node.ID, time.Since(start)) return nil } func (c *Cluster) copyDirToNode(node *node, src, dest string) error { start := time.Now() - c.t.Logf("Copying dir %s to node %s at %s", src, node.ID, dest) + c.logf("Copying dir %s to node %s at %s", src, node.ID, dest) _, err := os.Stat(src) if err != nil { @@ -447,7 +474,7 @@ func (c *Cluster) copyDirToNode(node *node, src, dest string) error { return fmt.Errorf("run command: %v, stderr: %s", err, stderr) } - c.t.Logf(" -> Copied dir %s to node %s in %s", src, node.ID, time.Since(start)) + c.logf(" -> Copied dir %s to node %s in %s", src, node.ID, time.Since(start)) return nil } @@ -459,6 +486,64 @@ func (c *Cluster) mkdirOnNode(node *node, dir string) error { return nil } +func (c *Cluster) generateSupportBundle(envs ...map[string]string) { + wg := sync.WaitGroup{} + wg.Add(len(c.nodes)) + + for i, node := range c.nodes { + go func(i int, wg *sync.WaitGroup) { + defer wg.Done() + c.logf("generating host support bundle from node %s", node.ID) + line := []string{"collect-support-bundle-host.sh"} + if stdout, stderr, err := c.runCommandOnNode(node, "root", line, envs...); err != nil { + c.logf("stdout: %s", stdout) + c.logf("stderr: %s", stderr) + c.logf("fail to generate support bundle from node %s: %v", node.ID, err) + return + } + + c.logf("copying host support bundle from node %s to local machine", node.ID) + if err := c.copyFileFromNode(node, "/root/host.tar.gz", fmt.Sprintf("support-bundle-host-%d.tar.gz", i)); err != nil { + c.logf("fail to copy host support bundle from node %s to local machine: %v", node.ID, err) + } + }(i, &wg) + } + + node := c.nodes[c.supportBundleNodeIndex] + c.logf("generating cluster support bundle from node %s", node.ID) + line := []string{"collect-support-bundle-cluster.sh"} + if stdout, stderr, err := c.runCommandOnNode(node, "root", line, envs...); err != nil { + c.logf("stdout: %s", stdout) + c.logf("stderr: %s", stderr) + c.logf("fail to generate cluster support from node %s bundle: %v", node.ID, err) + return + } + + c.logf("copying cluster support bundle from node %s to local machine", node.ID) + if err := c.copyFileFromNode(node, "/root/cluster.tar.gz", "support-bundle-cluster.tar.gz"); err != nil { + c.logf("fail to copy cluster support bundle from node %s to local machine: %v", node.ID, err) + } + + wg.Wait() +} + +func (c *Cluster) copyPlaywrightReport() { + line := []string{"tar", "-czf", "playwright-report.tar.gz", "-C", "/automation/playwright/playwright-report", "."} + c.logf("compressing playwright report on proxy node") + if _, _, err := c.runCommandOnNode(c.proxyNode, "root", line); err != nil { + c.logf("fail to compress playwright report on proxy node: %v", err) + return + } + c.logf("copying playwright report from proxy node to local machine") + if err := c.copyFileFromNode(c.proxyNode, "/root/playwright-report.tar.gz", "playwright-report.tar.gz"); err != nil { + c.logf("fail to copy playwright report from proxy node to local machine: %v", err) + } +} + +func (c *Cluster) logf(format string, args ...any) { + c.t.Logf("%s: "+format, append([]any{time.Now().Format(time.RFC3339)}, args...)...) +} + func sshConnectionArgs(node *node, sshUser string, isSCP bool) []string { args := []string{"-o", "StrictHostKeyChecking=no"} From 2e79430979a24fbe316c643e8e12b5a23f9b8c64 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 28 Mar 2025 14:56:17 -0700 Subject: [PATCH 14/31] f --- e2e/cluster/cmx/cluster.go | 2 +- e2e/install_test.go | 3 ++- e2e/shared.go | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 129590d68..cf5f41622 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -281,7 +281,7 @@ func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, st line := []string{"playwright.sh", testName} line = append(line, args...) env := map[string]string{ - "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.1", "30003")), + "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.2", "30003")), } stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", line, env) if err != nil { diff --git a/e2e/install_test.go b/e2e/install_test.go index 78686f5b3..83a5a7144 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2296,7 +2296,8 @@ func TestInstallWithPrivateCAs(t *testing.T) { } installSingleNodeWithOptions(t, tc, installOptions{ - privateCA: "/tmp/ca.crt", + networkInterface: "tailscale0", + privateCA: "/tmp/ca.crt", }) t.Logf("%s: deploying app", time.Now().Format(time.RFC3339)) diff --git a/e2e/shared.go b/e2e/shared.go index 983da0b74..826f48397 100644 --- a/e2e/shared.go +++ b/e2e/shared.go @@ -23,6 +23,7 @@ type installOptions struct { noProxy string privateCA string configValuesFile string + networkInterface string withEnv map[string]string } @@ -84,6 +85,9 @@ func installSingleNodeWithOptions(t *testing.T, tc cluster.Cluster, opts install if opts.configValuesFile != "" { line = append(line, "--config-values", opts.configValuesFile) } + if opts.networkInterface != "" { + line = append(line, "--network-interface", opts.networkInterface) + } t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339)) if stdout, stderr, err := tc.RunCommandOnNode(0, line, opts.withEnv); err != nil { From 815a4903808d5c98e0e1e0f54b4bf16f17c18aa0 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 05:21:41 -0700 Subject: [PATCH 15/31] f --- .github/actions/e2e/action.yml | 1 + e2e/cluster/cmx/cluster.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 4a7712cf7..9933a16a1 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -137,6 +137,7 @@ runs: export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }} export EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION=${{ inputs.upgrade-target-ec-version }} export SKIP_LXD_CLEANUP=true + export SKIP_CMX_CLEANUP=true make e2e-test TEST_NAME=${{ inputs.test-name }} - name: Troubleshoot if: ${{ !cancelled() }} diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index cf5f41622..c1e81cae9 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -291,6 +291,11 @@ func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, st } func (c *Cluster) destroy() { + if os.Getenv("SKIP_CMX_CLEANUP") != "" { + c.t.Logf("Skipping CMX cleanup") + return + } + if c.gid != "" { // Best effort cleanup c.logf("Cleaning up nodes") From b28f5ead4a67029609e8de69d0ecaa6191efb052 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 05:59:30 -0700 Subject: [PATCH 16/31] f --- e2e/cluster/cmx/cluster.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index c1e81cae9..c15b5eb9c 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -112,6 +112,12 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access on proxy node: %v", err) } + c.logf("Setting timezone on proxy node") + err = c.setTimezoneOnNode(c.proxyNode) + if err != nil { + return fmt.Errorf("set timezone on proxy node: %v", err) + } + c.logf("Copying dirs to proxy node") err = c.copyDirsToNode(c.proxyNode) if err != nil { @@ -144,6 +150,12 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access: %v", err) } + c.logf("Setting timezone on node %s", node.ID) + err = c.setTimezoneOnNode(node) + if err != nil { + return fmt.Errorf("set timezone on node %s: %v", node.ID, err) + } + c.logf("Copying files to node %s", node.ID) err = c.copyFilesToNode(node, input) if err != nil { @@ -355,6 +367,14 @@ func (c *Cluster) enableSSHAccessOnNode(node *node) error { return nil } +func (c *Cluster) setTimezoneOnNode(node *node) error { + _, stderr, err := c.runCommandOnNode(node, "root", []string{"timedatectl", "set-timezone", "Etc/UTC"}) + if err != nil { + return fmt.Errorf("set timezone on node %s: %v, stderr: %s", node.ID, err, stderr) + } + return nil +} + func (c *Cluster) copyFilesToNode(node *node, in ClusterInput) error { files := map[string]string{ in.LicensePath: "/assets/license.yaml", //0644 From f89d8adb88654f595544ea3f1d3fb35878f7aac4 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 06:42:26 -0700 Subject: [PATCH 17/31] f --- e2e/cluster/cmx/cluster.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index c15b5eb9c..4052462d6 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -33,6 +33,7 @@ type Cluster struct { gid string networkID string nodes []*node + nodePrivateIPs []string proxyNode *node supportBundleNodeIndex int } @@ -140,7 +141,9 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { eg = errgroup.Group{} - for _, node := range c.nodes { + c.nodePrivateIPs = make([]string, len(c.nodes)) + + for i, node := range c.nodes { eg.Go(func() error { start := time.Now() @@ -150,6 +153,9 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access: %v", err) } + c.logf("Discovering private IP for node %s", node.ID) + c.nodePrivateIPs[i] = c.discoverNodePrivateIP(node) + c.logf("Setting timezone on node %s", node.ID) err = c.setTimezoneOnNode(node) if err != nil { @@ -293,7 +299,7 @@ func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, st line := []string{"playwright.sh", testName} line = append(line, args...) env := map[string]string{ - "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort("100.64.0.2", "30003")), + "BASE_URL": fmt.Sprintf("http://%s", net.JoinHostPort(c.nodePrivateIPs[0], "30003")), } stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", line, env) if err != nil { @@ -367,6 +373,17 @@ func (c *Cluster) enableSSHAccessOnNode(node *node) error { return nil } +func (c *Cluster) discoverNodePrivateIP(node *node) string { + c.logf("Discovering private IP for node %s", node.ID) + ip, stderr, err := c.runCommandOnNode(node, "root", + []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `s/.*inet ([0-9.]+).*/\1/p`}, + ) + if err != nil { + c.t.Fatalf("Failed to get private IP for node %s: %v, stderr: %s", node.ID, err, stderr) + } + return ip +} + func (c *Cluster) setTimezoneOnNode(node *node) error { _, stderr, err := c.runCommandOnNode(node, "root", []string{"timedatectl", "set-timezone", "Etc/UTC"}) if err != nil { From 405f7235ee8ce302f303f5d9a16cf44e10c61120 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 07:09:35 -0700 Subject: [PATCH 18/31] f --- e2e/cluster/cmx/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 4052462d6..b16e4cf9b 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -376,7 +376,7 @@ func (c *Cluster) enableSSHAccessOnNode(node *node) error { func (c *Cluster) discoverNodePrivateIP(node *node) string { c.logf("Discovering private IP for node %s", node.ID) ip, stderr, err := c.runCommandOnNode(node, "root", - []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `s/.*inet ([0-9.]+).*/\1/p`}, + []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `'s/.*inet ([0-9.]+).*/\1/p'`}, ) if err != nil { c.t.Fatalf("Failed to get private IP for node %s: %v, stderr: %s", node.ID, err, stderr) From 92a0da8ad152381a8cded02d2490efbbbfd824e5 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 16:44:49 -0700 Subject: [PATCH 19/31] f --- e2e/cluster/cmx/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index b16e4cf9b..464771463 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -376,7 +376,7 @@ func (c *Cluster) enableSSHAccessOnNode(node *node) error { func (c *Cluster) discoverNodePrivateIP(node *node) string { c.logf("Discovering private IP for node %s", node.ID) ip, stderr, err := c.runCommandOnNode(node, "root", - []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `'s/.*inet ([0-9.]+).*/\1/p'`}, + []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `"s/.*inet ([0-9.]+).*/\1/p"`}, ) if err != nil { c.t.Fatalf("Failed to get private IP for node %s: %v, stderr: %s", node.ID, err, stderr) From 7ed692419b0c849756772175ac34c1fc48c2b0ee Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 20:09:29 -0700 Subject: [PATCH 20/31] f --- e2e/cluster/cmx/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 464771463..ace736b2c 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -381,7 +381,7 @@ func (c *Cluster) discoverNodePrivateIP(node *node) string { if err != nil { c.t.Fatalf("Failed to get private IP for node %s: %v, stderr: %s", node.ID, err, stderr) } - return ip + return strings.TrimSpace(ip) } func (c *Cluster) setTimezoneOnNode(node *node) error { From f870c877a3ea86e2d60e3ccc675c03ece77e4b6b Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 20:18:18 -0700 Subject: [PATCH 21/31] f --- e2e/cluster/cmx/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index ace736b2c..82d6afb83 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -342,7 +342,7 @@ func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, cmdArr = append(cmdArr, fmt.Sprintf("export %s=%s", k, v)) } } - cmdArr = append(cmdArr, fmt.Sprintf("sh -c '%s'", strings.Join(command, " "))) + cmdArr = append(cmdArr, strings.Join(command, " ")) args = append(args, strings.Join(cmdArr, "; ")) c.logf(" -> Running ssh command on node %s: %q", node.ID, args) @@ -376,7 +376,7 @@ func (c *Cluster) enableSSHAccessOnNode(node *node) error { func (c *Cluster) discoverNodePrivateIP(node *node) string { c.logf("Discovering private IP for node %s", node.ID) ip, stderr, err := c.runCommandOnNode(node, "root", - []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `"s/.*inet ([0-9.]+).*/\1/p"`}, + []string{"ip", "-f", "inet", "addr", "show", "tailscale0", "|", "sed", "-En", "-e", `'s/.*inet ([0-9.]+).*/\1/p'`}, ) if err != nil { c.t.Fatalf("Failed to get private IP for node %s: %v, stderr: %s", node.ID, err, stderr) From 0a051af308dbf5a52793cd314a14f549d1f96ef1 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Sun, 30 Mar 2025 22:06:25 -0700 Subject: [PATCH 22/31] f --- e2e/cluster/cmx/cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 82d6afb83..cf856346c 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -343,7 +343,7 @@ func (c *Cluster) runCommandOnNode(node *node, sshUser string, command []string, } } cmdArr = append(cmdArr, strings.Join(command, " ")) - args = append(args, strings.Join(cmdArr, "; ")) + args = append(args, strings.Join(cmdArr, " && ")) c.logf(" -> Running ssh command on node %s: %q", node.ID, args) cmd := exec.CommandContext(c.t.Context(), "ssh", args...) From 3fad9f45fac8a769e49ba44f76486853800345c2 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 06:19:51 -0700 Subject: [PATCH 23/31] f --- e2e/cluster/cmx/cluster.go | 21 +++++++++------------ e2e/install_test.go | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index cf856346c..872ef23fd 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -222,11 +222,10 @@ func (c *Cluster) RunCommandOnNode(node int, command []string, envs ...map[strin start := time.Now() c.logf("Running command on node %s: %s", c.nodes[node].ID, strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.nodes[node], "root", command, envs...) - if err != nil { - return stdout, stderr, err + if err == nil { + c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) } - c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) - return "", "", nil + return stdout, stderr, err } // RunCommandOnProxyNode executes a command on the proxy node as the root user @@ -234,11 +233,10 @@ func (c *Cluster) RunCommandOnProxyNode(command []string, envs ...map[string]str start := time.Now() c.logf("Running command on proxy node: %s", strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.proxyNode, "root", command, envs...) - if err != nil { - return stdout, stderr, err + if err == nil { + c.logf(" -> Command on proxy node completed in %s", time.Since(start)) } - c.logf(" -> Command on proxy node completed in %s", time.Since(start)) - return stdout, stderr, nil + return stdout, stderr, err } // RunRegularUserCommandOnNode executes a command on the specified node as a non-root user @@ -246,11 +244,10 @@ func (c *Cluster) RunRegularUserCommandOnNode(node int, command []string, envs . start := time.Now() c.logf("Running command on node %s as user %s: %s", c.nodes[node].ID, os.Getenv("REPLICATEDVM_SSH_USER"), strings.Join(command, " ")) stdout, stderr, err := c.runCommandOnNode(c.nodes[node], os.Getenv("REPLICATEDVM_SSH_USER"), command, envs...) - if err != nil { - return stdout, stderr, err + if err == nil { + c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) } - c.logf(" -> Command on node %s completed in %s", c.nodes[node].ID, time.Since(start)) - return stdout, stderr, nil + return stdout, stderr, err } // Cleanup removes the VM instance diff --git a/e2e/install_test.go b/e2e/install_test.go index 83a5a7144..f6de54066 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -2314,7 +2314,7 @@ func TestInstallWithPrivateCAs(t *testing.T) { var cm corev1.ConfigMap err = json.Unmarshal([]byte(stdout), &cm) - require.NoErrorf(t, err, "unable to unmarshal output to configmap: %q", stdout) + require.NoErrorf(t, err, "unable to unmarshal configmap output: %q", stdout) require.Contains(t, cm.Data, "ca_0.crt", "index ca_0.crt not found in ca secret") require.Equal(t, crtContent, cm.Data["ca_0.crt"], "content mismatch") From 8b01d71e51c1d72f61f244b4ec2d7d2d27513f5e Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 06:54:12 -0700 Subject: [PATCH 24/31] f --- e2e/cluster/cmx/cluster.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 872ef23fd..95878f75a 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -113,11 +113,11 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access on proxy node: %v", err) } - c.logf("Setting timezone on proxy node") - err = c.setTimezoneOnNode(c.proxyNode) - if err != nil { - return fmt.Errorf("set timezone on proxy node: %v", err) - } + // c.logf("Setting timezone on proxy node") + // err = c.setTimezoneOnNode(c.proxyNode) + // if err != nil { + // return fmt.Errorf("set timezone on proxy node: %v", err) + // } c.logf("Copying dirs to proxy node") err = c.copyDirsToNode(c.proxyNode) @@ -156,11 +156,11 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { c.logf("Discovering private IP for node %s", node.ID) c.nodePrivateIPs[i] = c.discoverNodePrivateIP(node) - c.logf("Setting timezone on node %s", node.ID) - err = c.setTimezoneOnNode(node) - if err != nil { - return fmt.Errorf("set timezone on node %s: %v", node.ID, err) - } + // c.logf("Setting timezone on node %s", node.ID) + // err = c.setTimezoneOnNode(node) + // if err != nil { + // return fmt.Errorf("set timezone on node %s: %v", node.ID, err) + // } c.logf("Copying files to node %s", node.ID) err = c.copyFilesToNode(node, input) From fbab18f935e76e5ffb8af87b6bca266bee801249 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 10:05:07 -0700 Subject: [PATCH 25/31] f --- e2e/cluster/cmx/cluster.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 95878f75a..872ef23fd 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -113,11 +113,11 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access on proxy node: %v", err) } - // c.logf("Setting timezone on proxy node") - // err = c.setTimezoneOnNode(c.proxyNode) - // if err != nil { - // return fmt.Errorf("set timezone on proxy node: %v", err) - // } + c.logf("Setting timezone on proxy node") + err = c.setTimezoneOnNode(c.proxyNode) + if err != nil { + return fmt.Errorf("set timezone on proxy node: %v", err) + } c.logf("Copying dirs to proxy node") err = c.copyDirsToNode(c.proxyNode) @@ -156,11 +156,11 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { c.logf("Discovering private IP for node %s", node.ID) c.nodePrivateIPs[i] = c.discoverNodePrivateIP(node) - // c.logf("Setting timezone on node %s", node.ID) - // err = c.setTimezoneOnNode(node) - // if err != nil { - // return fmt.Errorf("set timezone on node %s: %v", node.ID, err) - // } + c.logf("Setting timezone on node %s", node.ID) + err = c.setTimezoneOnNode(node) + if err != nil { + return fmt.Errorf("set timezone on node %s: %v", node.ID, err) + } c.logf("Copying files to node %s", node.ID) err = c.copyFilesToNode(node, input) From 11e4be19ae99b51afa2ca7cea86692a0643ab2fe Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 10:11:06 -0700 Subject: [PATCH 26/31] f --- dagger/go.mod | 2 +- go.mod | 6 ++---- kinds/go.mod | 3 ++- utils/go.mod | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dagger/go.mod b/dagger/go.mod index 25bc6b526..ad225a083 100644 --- a/dagger/go.mod +++ b/dagger/go.mod @@ -1,6 +1,6 @@ module dagger/embedded-cluster -go 1.23.2 +go 1.24.1 require ( github.com/99designs/gqlgen v0.17.57 diff --git a/go.mod b/go.mod index 230e41495..4b6ba693e 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/replicatedhq/embedded-cluster -go 1.24.0 - -toolchain go1.24.1 +go 1.24.1 require ( github.com/AlecAivazis/survey/v2 v2.3.7 @@ -43,6 +41,7 @@ require ( github.com/vmware-tanzu/velero v1.15.2 go.uber.org/multierr v1.11.0 golang.org/x/crypto v0.36.0 + golang.org/x/sync v0.12.0 golang.org/x/term v0.30.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -294,7 +293,6 @@ require ( go.opentelemetry.io/otel/trace v1.34.0 // indirect golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect golang.org/x/mod v0.23.0 // indirect - golang.org/x/sync v0.12.0 // indirect golang.org/x/tools v0.30.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.216.0 // indirect diff --git a/kinds/go.mod b/kinds/go.mod index bff91510d..45d3ede80 100644 --- a/kinds/go.mod +++ b/kinds/go.mod @@ -1,6 +1,7 @@ module github.com/replicatedhq/embedded-cluster/kinds -go 1.24.0 +go 1.24.1 + require ( github.com/k0sproject/k0s v1.30.7-0.20241029184556-a942e759e13b github.com/stretchr/testify v1.10.0 diff --git a/utils/go.mod b/utils/go.mod index c942b34e6..76ce6b1a2 100644 --- a/utils/go.mod +++ b/utils/go.mod @@ -1,6 +1,6 @@ module github.com/replicatedhq/embedded-cluster/utils -go 1.24.0 +go 1.24.1 require github.com/stretchr/testify v1.10.0 From 9716627891e05c8dca4dbc65a50aeae17806afdc Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 10:12:42 -0700 Subject: [PATCH 27/31] f --- .github/workflows/ci.yaml | 110 +++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2543b1e6e..c48246cfc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -632,36 +632,36 @@ jobs: matrix: test: - TestPreflights - # - TestPreflightsNoexec - # - TestMaterialize - # - TestHostPreflightCustomSpec - # - TestHostPreflightInBuiltSpec - # - TestSingleNodeInstallation - # - TestSingleNodeInstallationAlmaLinux8 - # - TestSingleNodeInstallationDebian11 - # - TestSingleNodeInstallationDebian12 - # - TestSingleNodeInstallationCentos9Stream - # - TestSingleNodeUpgradePreviousStable - # - TestInstallFromReplicatedApp - # - TestUpgradeFromReplicatedApp - # - TestUpgradeEC18FromReplicatedApp - # - TestResetAndReinstall - # # - TestOldVersionUpgrade - # - TestInstallSnapshotFromReplicatedApp - # - TestMultiNodeInstallation - # - TestMultiNodeHAInstallation - # - TestSingleNodeDisasterRecovery - # - TestSingleNodeLegacyDisasterRecovery - # - TestSingleNodeResumeDisasterRecovery - # - TestMultiNodeHADisasterRecovery - # - TestSingleNodeInstallationNoopUpgrade - # - TestCustomCIDR - # - TestLocalArtifactMirror - # - TestMultiNodeReset - # - TestCollectSupportBundle - # - TestUnsupportedOverrides - # - TestHostCollectSupportBundleInCluster - # - TestInstallWithConfigValues + - TestPreflightsNoexec + - TestMaterialize + - TestHostPreflightCustomSpec + - TestHostPreflightInBuiltSpec + - TestSingleNodeInstallation + - TestSingleNodeInstallationAlmaLinux8 + - TestSingleNodeInstallationDebian11 + - TestSingleNodeInstallationDebian12 + - TestSingleNodeInstallationCentos9Stream + - TestSingleNodeUpgradePreviousStable + - TestInstallFromReplicatedApp + - TestUpgradeFromReplicatedApp + - TestUpgradeEC18FromReplicatedApp + - TestResetAndReinstall + # - TestOldVersionUpgrade + - TestInstallSnapshotFromReplicatedApp + - TestMultiNodeInstallation + - TestMultiNodeHAInstallation + - TestSingleNodeDisasterRecovery + - TestSingleNodeLegacyDisasterRecovery + - TestSingleNodeResumeDisasterRecovery + - TestMultiNodeHADisasterRecovery + - TestSingleNodeInstallationNoopUpgrade + - TestCustomCIDR + - TestLocalArtifactMirror + - TestMultiNodeReset + - TestCollectSupportBundle + - TestUnsupportedOverrides + - TestHostCollectSupportBundleInCluster + - TestInstallWithConfigValues steps: - name: Checkout uses: actions/checkout@v4 @@ -725,32 +725,32 @@ jobs: fail-fast: false matrix: test: - # - TestVersion - # - TestCommandsRequireSudo - # - TestResetAndReinstallAirgap - # - TestSingleNodeAirgapUpgrade - # - TestSingleNodeAirgapUpgradeConfigValues - # - TestSingleNodeAirgapUpgradeCustomCIDR - # - TestSingleNodeDisasterRecoveryWithProxy - # - TestProxiedEnvironment - # - TestProxiedCustomCIDR + - TestVersion + - TestCommandsRequireSudo + - TestResetAndReinstallAirgap + - TestSingleNodeAirgapUpgrade + - TestSingleNodeAirgapUpgradeConfigValues + - TestSingleNodeAirgapUpgradeCustomCIDR + - TestSingleNodeDisasterRecoveryWithProxy + - TestProxiedEnvironment + - TestProxiedCustomCIDR - TestInstallWithPrivateCAs - # - TestInstallWithMITMProxy - include: [] - # - test: TestMultiNodeAirgapUpgrade - # runner: embedded-cluster-2 - # - test: TestMultiNodeAirgapUpgradeSameK0s - # runner: embedded-cluster-2 - # - test: TestMultiNodeAirgapUpgradePreviousStable - # runner: embedded-cluster-2 - # - test: TestAirgapUpgradeFromEC18 - # runner: embedded-cluster-2 - # - test: TestSingleNodeAirgapDisasterRecovery - # runner: embedded-cluster-2 - # - test: TestMultiNodeAirgapHAInstallation - # runner: embedded-cluster-2 - # - test: TestMultiNodeAirgapHADisasterRecovery - # runner: embedded-cluster-2 + - TestInstallWithMITMProxy + include: + - test: TestMultiNodeAirgapUpgrade + runner: embedded-cluster-2 + - test: TestMultiNodeAirgapUpgradeSameK0s + runner: embedded-cluster-2 + - test: TestMultiNodeAirgapUpgradePreviousStable + runner: embedded-cluster-2 + - test: TestAirgapUpgradeFromEC18 + runner: embedded-cluster-2 + - test: TestSingleNodeAirgapDisasterRecovery + runner: embedded-cluster-2 + - test: TestMultiNodeAirgapHAInstallation + runner: embedded-cluster-2 + - test: TestMultiNodeAirgapHADisasterRecovery + runner: embedded-cluster-2 steps: - name: Checkout uses: actions/checkout@v4 From 876becb5f4fe90dc4c09668573d26454211b0b60 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 10:53:27 -0700 Subject: [PATCH 28/31] f --- dagger.json | 6 +++-- dagger/go.mod | 35 +++++++++++++-------------- dagger/go.sum | 66 +++++++++++++++++++++++++-------------------------- 3 files changed, 53 insertions(+), 54 deletions(-) diff --git a/dagger.json b/dagger.json index 44dbe98b6..65d4e7141 100644 --- a/dagger.json +++ b/dagger.json @@ -1,6 +1,8 @@ { "name": "embedded-cluster", - "engineVersion": "v0.15.3", - "sdk": "go", + "engineVersion": "v0.17.2", + "sdk": { + "source": "go" + }, "source": "dagger" } diff --git a/dagger/go.mod b/dagger/go.mod index ad225a083..36df246fd 100644 --- a/dagger/go.mod +++ b/dagger/go.mod @@ -1,24 +1,23 @@ module dagger/embedded-cluster -go 1.24.1 +go 1.23.6 require ( - github.com/99designs/gqlgen v0.17.57 - github.com/Khan/genqlient v0.7.0 - github.com/vektah/gqlparser/v2 v2.5.20 - go.opentelemetry.io/otel v1.33.0 + github.com/99designs/gqlgen v0.17.68 + github.com/Khan/genqlient v0.8.0 + github.com/vektah/gqlparser/v2 v2.5.23 + go.opentelemetry.io/otel v1.34.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 go.opentelemetry.io/otel/log v0.8.0 - go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/sdk v1.34.0 go.opentelemetry.io/otel/sdk/log v0.8.0 - go.opentelemetry.io/otel/trace v1.33.0 + go.opentelemetry.io/otel/trace v1.34.0 go.opentelemetry.io/proto/otlp v1.3.1 - golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f - golang.org/x/sync v0.10.0 - google.golang.org/grpc v1.68.0 + golang.org/x/sync v0.12.0 + google.golang.org/grpc v1.71.0 ) require go.opentelemetry.io/auto/sdk v1.1.0 // indirect @@ -36,14 +35,14 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 - go.opentelemetry.io/otel/sdk/metric v1.32.0 - golang.org/x/net v0.34.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect - google.golang.org/protobuf v1.36.1 // indirect + go.opentelemetry.io/otel/metric v1.34.0 + go.opentelemetry.io/otel/sdk/metric v1.34.0 + golang.org/x/net v0.37.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/protobuf v1.36.5 // indirect ) replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 diff --git a/dagger/go.sum b/dagger/go.sum index caa199d24..c45899b64 100644 --- a/dagger/go.sum +++ b/dagger/go.sum @@ -1,7 +1,7 @@ -github.com/99designs/gqlgen v0.17.57 h1:Ak4p60BRq6QibxY0lEc0JnQhDurfhxA67sp02lMjmPc= -github.com/99designs/gqlgen v0.17.57/go.mod h1:Jx61hzOSTcR4VJy/HFIgXiQ5rJ0Ypw8DxWLjbYDAUw0= -github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= -github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= +github.com/99designs/gqlgen v0.17.68 h1:vH6jTShCv7sgz1ejXEDNqho7KWlA4ZwSWzVsxyhypAM= +github.com/99designs/gqlgen v0.17.68/go.mod h1:fvCiqQAu2VLhKXez2xFvLmE47QgAPf/KTPN5XQ4rsHQ= +github.com/Khan/genqlient v0.8.0 h1:Hd1a+E1CQHYbMEKakIkvBH3zW0PWEeiX6Hp1i2kP2WE= +github.com/Khan/genqlient v0.8.0/go.mod h1:hn70SpYjWteRGvxTwo0kfaqg4wxvndECGkfa1fdDdYI= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -37,12 +37,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vektah/gqlparser/v2 v2.5.20 h1:kPaWbhBntxoZPaNdBaIPT1Kh0i1b/onb5kXgEdP5JCo= -github.com/vektah/gqlparser/v2 v2.5.20/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM= +github.com/vektah/gqlparser/v2 v2.5.23 h1:PurJ9wpgEVB7tty1seRUwkIDa/QH5RzkzraiKIjKLfA= +github.com/vektah/gqlparser/v2 v2.5.23/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= -go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 h1:WzNab7hOOLzdDF/EoWCt4glhrbMPVMOO5JYTmpz36Ls= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0/go.mod h1:hKvJwTzJdp90Vh7p6q/9PAOd55dI6WA6sWj62a/JvSs= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 h1:S+LdBGiQXtJdowoJoQPEtI52syEP/JYBUpjO49EQhV8= @@ -59,38 +59,36 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI= go.opentelemetry.io/otel/log v0.8.0 h1:egZ8vV5atrUWUbnSsHn6vB8R21G2wrKqNiDt3iWertk= go.opentelemetry.io/otel/log v0.8.0/go.mod h1:M9qvDdUTRCopJcGRKg57+JSQ9LgLBrwwfC32epk5NX8= -go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= -go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= go.opentelemetry.io/otel/sdk/log v0.8.0 h1:zg7GUYXqxk1jnGF/dTdLPrK06xJdrXgqgFLnI4Crxvs= go.opentelemetry.io/otel/sdk/log v0.8.0/go.mod h1:50iXr0UVwQrYS45KbruFrEt4LvAdCaWWgIrsN3ZQggo= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= -go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= -golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f h1:M65LEviCfuZTfrfzwwEoxVtgvfkFkBUbFnRbxCXuXhU= -google.golang.org/genproto/googleapis/api v0.0.0-20241113202542-65e8d215514f/go.mod h1:Yo94eF2nj7igQt+TiJ49KxjIH8ndLYPZMIRSiRcEbg0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f h1:C1QccEa9kUwvMgEUORqQD9S17QesQijxjZ84sO82mfo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 0c2b59d2006c5de41a0a97df0a8daa27ecfdfbd2 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 10:59:12 -0700 Subject: [PATCH 29/31] f --- e2e/README.md | 3 ++- tests/dryrun/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 259c052ef..b09b09d69 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -3,7 +3,8 @@ ### Buildting the release for E2E tests ```bash -./scripts/build-and-release.sh ARCH=amd64 APP_VERSION="appver-dev-local-$USER" +./scripts/build-and-release.sh ARCH=amd64 \ + APP_VERSION="appver-dev-local-$USER" ``` ### Running individual CMX tests locally diff --git a/tests/dryrun/Dockerfile b/tests/dryrun/Dockerfile index edb7e8ad3..4b34b8b9d 100644 --- a/tests/dryrun/Dockerfile +++ b/tests/dryrun/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.0-alpine AS build +FROM golang:1.24-alpine AS build RUN apk add --no-cache ca-certificates curl git make bash From 8f052bf6e0a1b415ee9ec4af6be8c29cee0b0493 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 31 Mar 2025 11:03:27 -0700 Subject: [PATCH 30/31] f --- e2e/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index b09b09d69..173d0e019 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -3,8 +3,9 @@ ### Buildting the release for E2E tests ```bash -./scripts/build-and-release.sh ARCH=amd64 \ - APP_VERSION="appver-dev-local-$USER" +export ARCH=amd64 +export APP_VERSION="appver-dev-local-$USER" +./scripts/build-and-release.sh ``` ### Running individual CMX tests locally From 52ba546b324ac570e777740db4f527676a82f328 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 1 Apr 2025 12:41:52 -0700 Subject: [PATCH 31/31] f --- e2e/cluster/cmx/cluster.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/e2e/cluster/cmx/cluster.go b/e2e/cluster/cmx/cluster.go index 872ef23fd..7513df203 100644 --- a/e2e/cluster/cmx/cluster.go +++ b/e2e/cluster/cmx/cluster.go @@ -113,12 +113,6 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { return fmt.Errorf("enable ssh access on proxy node: %v", err) } - c.logf("Setting timezone on proxy node") - err = c.setTimezoneOnNode(c.proxyNode) - if err != nil { - return fmt.Errorf("set timezone on proxy node: %v", err) - } - c.logf("Copying dirs to proxy node") err = c.copyDirsToNode(c.proxyNode) if err != nil { @@ -156,12 +150,6 @@ func NewCluster(ctx context.Context, input ClusterInput) *Cluster { c.logf("Discovering private IP for node %s", node.ID) c.nodePrivateIPs[i] = c.discoverNodePrivateIP(node) - c.logf("Setting timezone on node %s", node.ID) - err = c.setTimezoneOnNode(node) - if err != nil { - return fmt.Errorf("set timezone on node %s: %v", node.ID, err) - } - c.logf("Copying files to node %s", node.ID) err = c.copyFilesToNode(node, input) if err != nil { @@ -381,14 +369,6 @@ func (c *Cluster) discoverNodePrivateIP(node *node) string { return strings.TrimSpace(ip) } -func (c *Cluster) setTimezoneOnNode(node *node) error { - _, stderr, err := c.runCommandOnNode(node, "root", []string{"timedatectl", "set-timezone", "Etc/UTC"}) - if err != nil { - return fmt.Errorf("set timezone on node %s: %v, stderr: %s", node.ID, err, stderr) - } - return nil -} - func (c *Cluster) copyFilesToNode(node *node, in ClusterInput) error { files := map[string]string{ in.LicensePath: "/assets/license.yaml", //0644