Skip to content

use EC kinds for the EC node join response, and include the app version #5274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/replicatedhq/kots

go 1.24.0
go 1.24.1

require (
cloud.google.com/go/storage v1.45.0
Expand Down Expand Up @@ -49,7 +49,7 @@ require (
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250411154749-d20d2f980f0c
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250415224730-0f6eb6643335
github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a
github.com/replicatedhq/kurlkinds v1.5.0
github.com/replicatedhq/troubleshoot v0.117.0
Expand Down Expand Up @@ -414,6 +414,7 @@ require (
github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/logutils v1.0.0 // indirect
github.com/k0sproject/dig v0.2.0 // indirect
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 // indirect
github.com/miekg/dns v1.1.63 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1545,6 +1545,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
github.com/k0sproject/dig v0.2.0 h1:cNxEIl96g9kqSMfPSZLhpnZ0P8bWXKv08nxvsMHop5w=
github.com/k0sproject/dig v0.2.0/go.mod h1:rBcqaQlJpcKdt2x/OE/lPvhGU50u/e95CSm5g/r4s78=
github.com/k0sproject/k0s v1.30.10-0.20250117153350-dcf3c22bb568 h1:JSfvTBrsNMWDISDUMVRZV6hP5eRusBS6d0Gv2lA4lSA=
github.com/k0sproject/k0s v1.30.10-0.20250117153350-dcf3c22bb568/go.mod h1:Nmj+slwFht6ile7OHHGiSrcRRGmrA9U9PzjnG9/6gc0=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
Expand Down Expand Up @@ -1861,8 +1863,8 @@ github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0
github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250411154749-d20d2f980f0c h1:QoAn+gFKypZPuybDUIzYQt/hzSLOMwTab/csazCAZQ8=
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250411154749-d20d2f980f0c/go.mod h1:DZVH5BSrkKaZPYO6psQYRZgzPdwaxrh8CpJYW95D76E=
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250415224730-0f6eb6643335 h1:PHC8qBJWV7+gfj1icO6dcM3eU5ofURdpcRdr2aj6sDw=
github.com/replicatedhq/embedded-cluster/kinds v1.15.1-0.20250415224730-0f6eb6643335/go.mod h1:+f76CfnrG1XqQyRvRaJ0+xMkRP9uvlWjx4hFySfnfnw=
github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a h1:aNZ7qcuEmPGIUIIfxF7c0sdKR2+zL2vc5r2V8j8a49I=
github.com/replicatedhq/kotskinds v0.0.0-20250411153224-089dbeb7ba2a/go.mod h1:QjhIUu3+OmHZ09u09j3FCoTt8F3BYtQglS+OLmftu9I=
github.com/replicatedhq/kurlkinds v1.5.0 h1:zZ0PKNeh4kXvSzVGkn62DKTo314GxhXg1TSB3azURMc=
Expand Down
45 changes: 32 additions & 13 deletions pkg/handlers/embedded_cluster_node_join_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import (
"fmt"
"net/http"

ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/google/uuid"
"github.com/replicatedhq/embedded-cluster/kinds/types/join"

"github.com/replicatedhq/kots/pkg/embeddedcluster"
"github.com/replicatedhq/kots/pkg/k8sutil"
Expand All @@ -19,16 +20,6 @@ type GenerateEmbeddedClusterNodeJoinCommandResponse struct {
Command []string `json:"command"`
}

type GetEmbeddedClusterNodeJoinCommandResponse struct {
ClusterID string `json:"clusterID"`
K0sJoinCommand string `json:"k0sJoinCommand"`
K0sToken string `json:"k0sToken"`
EmbeddedClusterVersion string `json:"embeddedClusterVersion"`
AirgapRegistryAddress string `json:"airgapRegistryAddress"`
TCPConnectionsRequired []string `json:"tcpConnectionsRequired"`
InstallationSpec ecv1beta1.InstallationSpec `json:"installationSpec,omitempty"`
}

type GenerateEmbeddedClusterNodeJoinCommandRequest struct {
Roles []string `json:"roles"`
}
Expand Down Expand Up @@ -178,13 +169,41 @@ func (h *Handler) GetEmbeddedClusterNodeJoinCommand(w http.ResponseWriter, r *ht
return
}

JSON(w, http.StatusOK, GetEmbeddedClusterNodeJoinCommandResponse{
ClusterID: install.Spec.ClusterID,
clusterUUID, err := uuid.Parse(install.Spec.ClusterID)
if err != nil {
logger.Error(fmt.Errorf("failed to parse cluster id: %w", err))
w.WriteHeader(http.StatusInternalServerError)
return
}

var currentAppVersionLabel string
// attempt to get the current app version label from the installed app
installedApps, err := store.GetStore().ListInstalledApps()
if err == nil && len(installedApps) > 0 {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think you should fail if err != nil here, as that won't fail if there are no apps installed.

// "CurrentSequence" is the latest available version of the app in a non-embedded cluster.
// However, in an embedded cluster, the "CurrentSequence" is also the currently deployed version of the app.
// This is because EC uses the new upgrade flow, which only creates a new app version when
// the app version gets deployed. And because rollbacks are not supported in embedded cluster yet.
appVersion, err := store.GetStore().GetAppVersion(installedApps[0].ID, installedApps[0].CurrentSequence)
if err != nil {
logger.Error(fmt.Errorf("failed to get app version: %w", err))
w.WriteHeader(http.StatusInternalServerError)
return
}
currentAppVersionLabel = appVersion.VersionLabel
} else {
// if there are no installed apps, we can't get the current app version label
logger.Info("no installed apps found")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think this should fail. same if the list of installed apps != 1

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can join nodes before installing an app - or at least that's my understanding of the sequencing

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the app would be installed at this point. the meaning of installed here is a bit different... it's not that the app got deployed, it means that the license (and airgap bundle) have been processed.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha
I'd still rather fall back to "we don't include the app version in the response" instead of "you can't join nodes" though - this is an extra layer of safety check, but far from the only one

}

JSON(w, http.StatusOK, join.JoinCommandResponse{
ClusterID: clusterUUID,
K0sJoinCommand: k0sJoinCommand,
K0sToken: k0sToken,
EmbeddedClusterVersion: ecVersion,
AirgapRegistryAddress: airgapRegistryAddress,
TCPConnectionsRequired: endpoints,
InstallationSpec: install.Spec,
AppVersionLabel: currentAppVersionLabel,
})
}
109 changes: 100 additions & 9 deletions pkg/handlers/embedded_cluster_node_join_command_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ import (
"time"

gomock "github.com/golang/mock/gomock"
"github.com/google/uuid"
embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/kinds/types/join"
versionTypes "github.com/replicatedhq/kots/pkg/api/version/types"
appTypes "github.com/replicatedhq/kots/pkg/app/types"
"github.com/replicatedhq/kots/pkg/handlers/kubeclient"
"github.com/replicatedhq/kots/pkg/store"
mockstore "github.com/replicatedhq/kots/pkg/store/mock"
Expand All @@ -33,13 +37,15 @@ type testNodeJoinCommandHarness struct {
token string
getRoles func(t *testing.T, token string) ([]string, error)
embeddedClusterID string
validateBody func(t *testing.T, h *testNodeJoinCommandHarness, r *GetEmbeddedClusterNodeJoinCommandResponse)
appVersionLabel string
validateBody func(t *testing.T, h *testNodeJoinCommandHarness, r *join.JoinCommandResponse)
}

func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
scheme := runtime.NewScheme()
corev1.AddToScheme(scheme)
embeddedclusterv1beta1.AddToScheme(scheme)
ecUUID := uuid.New().String()

tests := []testNodeJoinCommandHarness{
{
Expand All @@ -50,15 +56,15 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
{
name: "store returns error",
httpStatus: http.StatusInternalServerError,
embeddedClusterID: "cluster-id",
embeddedClusterID: ecUUID,
getRoles: func(*testing.T, string) ([]string, error) {
return nil, fmt.Errorf("some error")
},
},
{
name: "store gets passed the provided token",
httpStatus: http.StatusInternalServerError,
embeddedClusterID: "cluster-id",
embeddedClusterID: ecUUID,
token: "some-token",
getRoles: func(t *testing.T, token string) ([]string, error) {
require.Equal(t, "some-token", token)
Expand All @@ -68,8 +74,9 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
{
name: "bootstrap token secret creation succeeds and it matches returned K0SToken",
httpStatus: http.StatusOK,
embeddedClusterID: "cluster-id",
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *GetEmbeddedClusterNodeJoinCommandResponse) {
embeddedClusterID: ecUUID,
appVersionLabel: "test-app-version",
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *join.JoinCommandResponse) {
req := require.New(t)
// Check that a secret was created with the cluster bootstrap token
var secrets corev1.SecretList
Expand All @@ -91,7 +98,11 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
decompressed, err := util.GunzipData(decodedK0SToken)
req.NoError(err)

require.Containsf(t, string(decompressed), fmt.Sprintf("token: %s", expectedToken), "expected K0sToken:\n%s\nto contain the generated bootstrap token: %s", string(decompressed), expectedToken)
req.Containsf(string(decompressed), fmt.Sprintf("token: %s", expectedToken), "expected K0sToken:\n%s\nto contain the generated bootstrap token: %s", string(decompressed), expectedToken)

// returned embedded cluster and app version should match the expected values
req.Equal(r.AppVersionLabel, "test-app-version")
req.Equal(r.ClusterID.String(), ecUUID)
},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
Expand All @@ -100,6 +111,7 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
ClusterID: ecUUID,
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Expand Down Expand Up @@ -144,8 +156,9 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
{
name: "tcp connections required are returned based on the controller role provided",
httpStatus: http.StatusOK,
embeddedClusterID: "cluster-id",
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *GetEmbeddedClusterNodeJoinCommandResponse) {
embeddedClusterID: ecUUID,
appVersionLabel: "test-app-version",
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *join.JoinCommandResponse) {
req := require.New(t)

req.Equal([]string{
Expand All @@ -166,6 +179,7 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
ClusterID: ecUUID,
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Expand Down Expand Up @@ -227,6 +241,65 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
},
).Build(),
},
{
name: "If there is no installed app, the app version label should be empty",
httpStatus: http.StatusOK,
embeddedClusterID: ecUUID,
validateBody: func(t *testing.T, h *testNodeJoinCommandHarness, r *join.JoinCommandResponse) {
req := require.New(t)
// returned embedded cluster and app version should match the expected values
req.Equal(r.AppVersionLabel, "")
req.Equal(r.ClusterID.String(), ecUUID)
},
kbClient: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&embeddedclusterv1beta1.Installation{
ObjectMeta: metav1.ObjectMeta{
Name: time.Now().Format("20060102150405"),
},
Spec: embeddedclusterv1beta1.InstallationSpec{
BinaryName: "my-app",
ClusterID: ecUUID,
Config: &embeddedclusterv1beta1.ConfigSpec{
Version: "v1.100.0",
Roles: embeddedclusterv1beta1.Roles{
Controller: embeddedclusterv1beta1.NodeRole{
Name: "controller-role",
},
},
},
},
},
&corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-root-ca.crt",
Namespace: "kube-system",
},
Data: map[string]string{"ca.crt": "some-ca-cert"},
},
&corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "controller 1",
Labels: map[string]string{
"node-role.kubernetes.io/control-plane": "true",
},
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: corev1.NodeReady,
Status: corev1.ConditionTrue,
},
},
Addresses: []corev1.NodeAddress{
{
Type: corev1.NodeInternalIP,
Address: "192.168.0.100",
},
},
},
},
).Build(),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand All @@ -251,6 +324,24 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
return []string{"controller-role", "worker-role"}, nil
})

if test.appVersionLabel != "" {
mockStore.EXPECT().ListInstalledApps().AnyTimes().DoAndReturn(func() ([]*appTypes.App, error) {
return []*appTypes.App{
{
ID: "test-app-id",
CurrentSequence: 1,
},
}, nil
})
mockStore.EXPECT().GetAppVersion("test-app-id", int64(1)).AnyTimes().DoAndReturn(func(appID string, sequence int64) (*versionTypes.AppVersion, error) {
return &versionTypes.AppVersion{
VersionLabel: test.appVersionLabel,
}, nil
})
} else if test.httpStatus == http.StatusOK {
mockStore.EXPECT().ListInstalledApps().AnyTimes().Return([]*appTypes.App{}, nil)
}

// There's an early check in the handler for the presence of `EMBEDDED_CLUSTER_ID` env var
// so we need to set it here whenever the test requires it
if test.embeddedClusterID != "" {
Expand All @@ -274,7 +365,7 @@ func TestGetEmbeddedClusterNodeJoinCommand(t *testing.T) {
}

// Run the body validation function if provided
var body GetEmbeddedClusterNodeJoinCommandResponse
var body join.JoinCommandResponse
req.NoError(json.NewDecoder(response.Body).Decode(&body))
if test.validateBody != nil {
test.validateBody(t, &test, &body)
Expand Down
Loading