Skip to content

Commit 147f92c

Browse files
feat(operator): Mount CA into Upgrade Job Container (#2183)
* feat(velero): add support for mitm proxy * f * f * f * f * f * f * f * velero use ca from host * http proxy dryrun test * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * f * refactor host ca retrieval * add unit tests * add dry run capability for adminconsole addon * make adminconsole unit test consistent with other addons * remove privateCa from admin console * update adminconsole version and add integration test * add tests for cabundle to dryrun * update TestInstallWithPrivateCAs e2e test * fix dry run tests to make sure they use dry run client * more debug * more debug * fix tests * fix tests * remove debug values * debug integration test * fix integration test * move cert generation to bash * debug privateca e2e test * debug privateca e2e test * re-add privateca in order to get operator working correctly for mitm tests * fix create configmap * remove uneeded e2e test * consolidate unit test * remove privateca from static values * remove redundant dry run test * retreive hostcabundlepath from runtime config * add debug log statements * bump kots version for latest kots kinds * remove debug logging --------- Co-authored-by: Ethan Mosbaugh <[email protected]>
1 parent e1118b3 commit 147f92c

File tree

3 files changed

+261
-26
lines changed

3 files changed

+261
-26
lines changed

operator/pkg/upgrade/job.go

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ func CreateUpgradeJob(
4343
localArtifactMirrorImage, licenseID, appSlug, channelID, appVersion string,
4444
previousInstallVersion string,
4545
) error {
46+
log := controllerruntime.LoggerFrom(ctx)
4647
// check if the job already exists - if it does, we've already rolled out images and can return now
4748
job := &batchv1.Job{}
4849
err := cli.Get(ctx, client.ObjectKey{Namespace: upgradeJobNamespace, Name: fmt.Sprintf(upgradeJobName, in.Name)}, job)
@@ -118,10 +119,6 @@ func CreateUpgradeJob(
118119
Name: "JOB_NAMESPACE",
119120
Value: upgradeJobNamespace,
120121
},
121-
{
122-
Name: "SSL_CERT_DIR",
123-
Value: "/certs",
124-
},
125122
}
126123

127124
if in.Spec.Proxy != nil {
@@ -195,17 +192,6 @@ func CreateUpgradeJob(
195192
},
196193
},
197194
},
198-
{
199-
Name: "private-cas",
200-
VolumeSource: corev1.VolumeSource{
201-
ConfigMap: &corev1.ConfigMapVolumeSource{
202-
LocalObjectReference: corev1.LocalObjectReference{
203-
Name: "kotsadm-private-cas",
204-
},
205-
Optional: ptr.To(true),
206-
},
207-
},
208-
},
209195
{
210196
Name: "ec-charts-dir",
211197
VolumeSource: corev1.VolumeSource{
@@ -236,10 +222,6 @@ func CreateUpgradeJob(
236222
Name: "config",
237223
MountPath: "/config",
238224
},
239-
{
240-
Name: "private-cas",
241-
MountPath: "/certs",
242-
},
243225
{
244226
Name: "ec-charts-dir",
245227
MountPath: runtimeconfig.EmbeddedClusterChartsSubDirNoCreate(),
@@ -253,6 +235,48 @@ func CreateUpgradeJob(
253235
},
254236
}
255237

238+
// Add the host CA bundle volume, mount, and env var if it's available in the installation
239+
hostCABundlePath := ""
240+
if in.Spec.RuntimeConfig != nil {
241+
hostCABundlePath = in.Spec.RuntimeConfig.HostCABundlePath
242+
}
243+
244+
if hostCABundlePath != "" {
245+
log.Info("Using host CA bundle from installation", "path", hostCABundlePath)
246+
247+
// Add the CA bundle volume
248+
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
249+
Name: "host-ca-bundle",
250+
VolumeSource: corev1.VolumeSource{
251+
HostPath: &corev1.HostPathVolumeSource{
252+
Path: hostCABundlePath,
253+
Type: ptr.To(corev1.HostPathFileOrCreate),
254+
},
255+
},
256+
})
257+
258+
// Add the CA bundle mount
259+
job.Spec.Template.Spec.Containers[0].VolumeMounts = append(
260+
job.Spec.Template.Spec.Containers[0].VolumeMounts,
261+
corev1.VolumeMount{
262+
Name: "host-ca-bundle",
263+
MountPath: "/certs/ca-certificates.crt",
264+
},
265+
)
266+
267+
// Add the SSL_CERT_DIR environment variable
268+
job.Spec.Template.Spec.Containers[0].Env = append(
269+
job.Spec.Template.Spec.Containers[0].Env,
270+
corev1.EnvVar{
271+
Name: "SSL_CERT_DIR",
272+
Value: "/certs",
273+
},
274+
)
275+
} else {
276+
log.Info("No host CA bundle path found in installation, no CA bundle will be used")
277+
}
278+
279+
// Create the job with all configuration in place
256280
if err = cli.Create(ctx, job); err != nil {
257281
return fmt.Errorf("failed to create upgrade job: %w", err)
258282
}

operator/pkg/upgrade/job_test.go

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,3 +88,214 @@ func TestCreateUpgradeJob_NodeAffinity(t *testing.T) {
8888
assert.Equal(t, corev1.NodeSelectorOpExists, preferredTerms[0].Preference.MatchExpressions[0].Operator,
8989
"Node affinity operator should be 'Exists'")
9090
}
91+
92+
func TestCreateUpgradeJob_HostCABundle(t *testing.T) {
93+
// Test with HostCABundlePath set
94+
t.Run("with HostCABundlePath set", func(t *testing.T) {
95+
scheme := runtime.NewScheme()
96+
require.NoError(t, ecv1beta1.AddToScheme(scheme))
97+
require.NoError(t, batchv1.AddToScheme(scheme))
98+
require.NoError(t, corev1.AddToScheme(scheme))
99+
100+
// Version used for testing
101+
testVersion := "1.2.3"
102+
testCAPath := "/etc/ssl/certs/ca-certificates.crt"
103+
104+
// Create a minimal installation CR with RuntimeConfig.HostCABundlePath set
105+
installation := &ecv1beta1.Installation{
106+
ObjectMeta: metav1.ObjectMeta{
107+
Name: "test-installation",
108+
Namespace: "default",
109+
},
110+
Spec: ecv1beta1.InstallationSpec{
111+
BinaryName: "test-binary",
112+
Config: &ecv1beta1.ConfigSpec{
113+
Version: testVersion,
114+
Domains: ecv1beta1.Domains{
115+
ProxyRegistryDomain: "registry.example.com",
116+
},
117+
},
118+
RuntimeConfig: &ecv1beta1.RuntimeConfigSpec{
119+
HostCABundlePath: testCAPath,
120+
},
121+
},
122+
}
123+
124+
// Create a cached metadata for the test version
125+
// This avoids having to properly create a ConfigMap
126+
testMeta := types.ReleaseMetadata{
127+
Images: []string{"registry.example.com/embedded-cluster-operator-image:1.2.3"},
128+
}
129+
release.CacheMeta(testVersion, testMeta)
130+
131+
// Create a fake client with the installation
132+
cli := fake.NewClientBuilder().
133+
WithScheme(scheme).
134+
WithObjects(installation).
135+
Build()
136+
137+
// Call the function under test
138+
err := CreateUpgradeJob(
139+
context.Background(), cli, installation,
140+
"registry.example.com/local-artifact-mirror:1.2.3",
141+
"license-id", "app-slug", "channel-id", testVersion,
142+
"1.2.2",
143+
)
144+
require.NoError(t, err)
145+
146+
// Get the job that was created
147+
job := &batchv1.Job{}
148+
err = cli.Get(context.Background(), client.ObjectKey{
149+
Namespace: upgradeJobNamespace,
150+
Name: "embedded-cluster-upgrade-test-installation",
151+
}, job)
152+
require.NoError(t, err)
153+
154+
// Verify that the host CA bundle volume exists
155+
var hostCABundleVolumeFound bool
156+
for _, volume := range job.Spec.Template.Spec.Volumes {
157+
if volume.Name == "host-ca-bundle" {
158+
hostCABundleVolumeFound = true
159+
// Verify the volume properties
160+
require.NotNil(t, volume.HostPath, "Host CA bundle volume should be a hostPath volume")
161+
assert.Equal(t, testCAPath, volume.HostPath.Path, "Host CA bundle path should match RuntimeConfig.HostCABundlePath")
162+
assert.Equal(t, corev1.HostPathFileOrCreate, *volume.HostPath.Type, "Host CA bundle type should be FileOrCreate")
163+
break
164+
}
165+
}
166+
assert.True(t, hostCABundleVolumeFound, "Host CA bundle volume should exist")
167+
168+
// Verify that the volume mount exists
169+
var hostCABundleMountFound bool
170+
for _, mount := range job.Spec.Template.Spec.Containers[0].VolumeMounts {
171+
if mount.Name == "host-ca-bundle" {
172+
hostCABundleMountFound = true
173+
// Verify the mount properties
174+
assert.Equal(t, "/certs/ca-certificates.crt", mount.MountPath, "Host CA bundle mount path should be correct")
175+
break
176+
}
177+
}
178+
assert.True(t, hostCABundleMountFound, "Host CA bundle mount should exist")
179+
180+
// Verify that the SSL_CERT_DIR environment variable exists
181+
var sslCertDirEnvFound bool
182+
for _, env := range job.Spec.Template.Spec.Containers[0].Env {
183+
if env.Name == "SSL_CERT_DIR" {
184+
sslCertDirEnvFound = true
185+
// Verify the env var value
186+
assert.Equal(t, "/certs", env.Value, "SSL_CERT_DIR value should be correct")
187+
break
188+
}
189+
}
190+
assert.True(t, sslCertDirEnvFound, "SSL_CERT_DIR environment variable should exist")
191+
192+
// Verify the "private-cas" volume does NOT exist
193+
var privateCasVolumeFound bool
194+
for _, volume := range job.Spec.Template.Spec.Volumes {
195+
if volume.Name == "private-cas" {
196+
privateCasVolumeFound = true
197+
break
198+
}
199+
}
200+
assert.False(t, privateCasVolumeFound, "private-cas volume should not exist")
201+
})
202+
203+
// Test without HostCABundlePath set
204+
t.Run("without HostCABundlePath set", func(t *testing.T) {
205+
scheme := runtime.NewScheme()
206+
require.NoError(t, ecv1beta1.AddToScheme(scheme))
207+
require.NoError(t, batchv1.AddToScheme(scheme))
208+
require.NoError(t, corev1.AddToScheme(scheme))
209+
210+
// Version used for testing
211+
testVersion := "1.2.3"
212+
213+
// Create a minimal installation CR without RuntimeConfig.HostCABundlePath
214+
installation := &ecv1beta1.Installation{
215+
ObjectMeta: metav1.ObjectMeta{
216+
Name: "test-installation",
217+
Namespace: "default",
218+
},
219+
Spec: ecv1beta1.InstallationSpec{
220+
BinaryName: "test-binary",
221+
Config: &ecv1beta1.ConfigSpec{
222+
Version: testVersion,
223+
Domains: ecv1beta1.Domains{
224+
ProxyRegistryDomain: "registry.example.com",
225+
},
226+
},
227+
// No RuntimeConfig or empty RuntimeConfig
228+
},
229+
}
230+
231+
// Create a cached metadata for the test version
232+
// This avoids having to properly create a ConfigMap
233+
testMeta := types.ReleaseMetadata{
234+
Images: []string{"registry.example.com/embedded-cluster-operator-image:1.2.3"},
235+
}
236+
release.CacheMeta(testVersion, testMeta)
237+
238+
// Create a fake client with the installation
239+
cli := fake.NewClientBuilder().
240+
WithScheme(scheme).
241+
WithObjects(installation).
242+
Build()
243+
244+
// Call the function under test
245+
err := CreateUpgradeJob(
246+
context.Background(), cli, installation,
247+
"registry.example.com/local-artifact-mirror:1.2.3",
248+
"license-id", "app-slug", "channel-id", testVersion,
249+
"1.2.2",
250+
)
251+
require.NoError(t, err)
252+
253+
// Get the job that was created
254+
job := &batchv1.Job{}
255+
err = cli.Get(context.Background(), client.ObjectKey{
256+
Namespace: upgradeJobNamespace,
257+
Name: "embedded-cluster-upgrade-test-installation",
258+
}, job)
259+
require.NoError(t, err)
260+
261+
// Verify that the host CA bundle volume does NOT exist
262+
var hostCABundleVolumeFound bool
263+
for _, volume := range job.Spec.Template.Spec.Volumes {
264+
if volume.Name == "host-ca-bundle" {
265+
hostCABundleVolumeFound = true
266+
break
267+
}
268+
}
269+
assert.False(t, hostCABundleVolumeFound, "Host CA bundle volume should not exist when HostCABundlePath is not set")
270+
271+
// Verify that the volume mount does NOT exist
272+
var hostCABundleMountFound bool
273+
for _, mount := range job.Spec.Template.Spec.Containers[0].VolumeMounts {
274+
if mount.Name == "host-ca-bundle" {
275+
hostCABundleMountFound = true
276+
break
277+
}
278+
}
279+
assert.False(t, hostCABundleMountFound, "Host CA bundle mount should not exist when HostCABundlePath is not set")
280+
281+
// Verify that the SSL_CERT_DIR environment variable does NOT exist
282+
var sslCertDirEnvFound bool
283+
for _, env := range job.Spec.Template.Spec.Containers[0].Env {
284+
if env.Name == "SSL_CERT_DIR" {
285+
sslCertDirEnvFound = true
286+
break
287+
}
288+
}
289+
assert.False(t, sslCertDirEnvFound, "SSL_CERT_DIR environment variable should not exist when HostCABundlePath is not set")
290+
291+
// Verify the "private-cas" volume does NOT exist
292+
var privateCasVolumeFound bool
293+
for _, volume := range job.Spec.Template.Spec.Volumes {
294+
if volume.Name == "private-cas" {
295+
privateCasVolumeFound = true
296+
break
297+
}
298+
}
299+
assert.False(t, privateCasVolumeFound, "private-cas volume should not exist")
300+
})
301+
}

pkg/addons/adminconsole/static/metadata.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,24 +5,24 @@
55
# $ make buildtools
66
# $ output/bin/buildtools update addon <addon name>
77
#
8-
version: 1.124.16-ec.4
8+
version: 1.124.16-ec.5
99
location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
1010
images:
1111
kotsadm:
1212
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm
1313
tag:
14-
amd64: v1.124.16-ec.4-amd64@sha256:25867454b00cae4c36dcb1cab30fe833227337fa698afe1d9450e85d0dda0461
15-
arm64: v1.124.16-ec.4-arm64@sha256:76d95bd8c53a9e7f3e170b927f847c6b308457dd83a7e37229c0f0770e40d0d6
14+
amd64: v1.124.16-ec.5-amd64@sha256:f4519dd15b4b978157e699ea16616fba646d7b599e8c579b0e83774c1e2e02bd
15+
arm64: v1.124.16-ec.5-arm64@sha256:53310cc5e666a8d1acafa4309a88aceedc134a86616f58306f1f665a89ddf9ed
1616
kotsadm-migrations:
1717
repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations
1818
tag:
19-
amd64: v1.124.16-ec.4-amd64@sha256:b0ac39c9cd9a727ab8e11738aa28ba8e3f5415fdf444b3d7e10dbdc47078b281
20-
arm64: v1.124.16-ec.4-arm64@sha256:7b6754b4e444436c429161a0b292f5c6afe2d9aa7a04083cab588a2ecdb5e87f
19+
amd64: v1.124.16-ec.5-amd64@sha256:1c3daf5c301fc6a9d64dc7f8776d40db9be5a0e82766ba21d2fd4daa35a525eb
20+
arm64: v1.124.16-ec.5-arm64@sha256:8139f3566f96f772bf52244c0f9ddaad94193a39e64b76ee700e02adad8b441a
2121
kurl-proxy:
2222
repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy
2323
tag:
24-
amd64: v1.124.16-ec.4-amd64@sha256:5296b3ce56fbf55eee420efc7e939514fc11e0e6c83ff1240288ecca36c47473
25-
arm64: v1.124.16-ec.4-arm64@sha256:35af6a5e0566416b5a9d06da20725fdbfaa10727f255cb31520715e422c220e3
24+
amd64: v1.124.16-ec.5-amd64@sha256:1a367bae52522e45eb6a30794d7a5796b6010703926524aca31e151eca382d8a
25+
arm64: v1.124.16-ec.5-arm64@sha256:d0629818c1bb38a2a876becdce3c94c4b96dfd9f9d21e4f3d7225703994518d0
2626
rqlite:
2727
repo: proxy.replicated.com/anonymous/kotsadm/rqlite
2828
tag:

0 commit comments

Comments
 (0)