Skip to content

Commit 3ed87ba

Browse files
laveryalaverya
authored
cache operator and kots binaries in the public files s3 bucket (#683)
* cache operator binary in the public files s3 bucket * fail on failing http codes * update operator import * use 'Install', not 'AddToScheme' * update embedded-cluster-kinds * cache kots bin, include artifact paths in metadata * back to addtoscheme * return k0s version * fix kots binary source * use correct kots binary * update operator to revert k0s upgrade * use existing KOTS_VERSION and KOTS_BINARY_URL_OVERRIDE makefile vars * fix merge * test artifact output
1 parent 664d3ae commit 3ed87ba

File tree

8 files changed

+111
-35
lines changed

8 files changed

+111
-35
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ ADMIN_CONSOLE_IMAGE_OVERRIDE =
99
ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
1010
EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
1111
EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
12-
EMBEDDED_OPERATOR_CHART_VERSION = 0.34.9
12+
EMBEDDED_OPERATOR_CHART_VERSION = 0.35.2
1313
EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
1414
EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
1515
OPENEBS_CHART_URL = https://openebs.github.io/openebs
@@ -39,7 +39,6 @@ LOCAL_ARTIFACT_MIRROR_IMAGE ?= registry.replicated.com/library/embedded-cluster-
3939
LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION = ${LOCAL_ARTIFACT_MIRROR_IMAGE}:$(subst +,-,$(VERSION))
4040
LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \
4141
-X github.com/replicatedhq/embedded-cluster/pkg/defaults.Version=$(VERSION) \
42-
-X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sBinaryURL=$(K0S_BINARY_SOURCE_OVERRIDE) \
4342
-X github.com/replicatedhq/embedded-cluster/pkg/defaults.TroubleshootVersion=$(TROUBLESHOOT_VERSION) \
4443
-X github.com/replicatedhq/embedded-cluster/pkg/defaults.KubectlVersion=$(KUBECTL_VERSION) \
4544
-X github.com/replicatedhq/embedded-cluster/pkg/defaults.LocalArtifactMirrorImage=$(LOCAL_ARTIFACT_MIRROR_IMAGE_LOCATION) \
@@ -48,6 +47,7 @@ LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$
4847
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.Version=$(ADMIN_CONSOLE_CHART_VERSION) \
4948
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.ImageOverride=$(ADMIN_CONSOLE_IMAGE_OVERRIDE) \
5049
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.MigrationsImageOverride=$(ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE) \
50+
-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.KotsVersion=$(KOTS_VERSION) \
5151
-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.ChartURL=$(EMBEDDED_OPERATOR_CHART_URL) \
5252
-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.ChartName=$(EMBEDDED_OPERATOR_CHART_NAME) \
5353
-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.Version=$(EMBEDDED_OPERATOR_CHART_VERSION) \

‎cmd/embedded-cluster/version.go

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"github.com/urfave/cli/v2"
1414

1515
"github.com/replicatedhq/embedded-cluster/pkg/addons"
16+
"github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole"
17+
"github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator"
1618
"github.com/replicatedhq/embedded-cluster/pkg/config"
1719
"github.com/replicatedhq/embedded-cluster/pkg/defaults"
1820
"github.com/replicatedhq/embedded-cluster/pkg/goods"
@@ -103,10 +105,16 @@ func gatherVersionMetadata() (*types.ReleaseMetadata, error) {
103105
return nil, fmt.Errorf("unable to get k0s binary sha256: %w", err)
104106
}
105107

108+
artifacts := map[string]string{
109+
"kots": fmt.Sprintf("kots-binaries/%s", adminconsole.KotsVersion),
110+
"operator": fmt.Sprintf("operator-binaries/%s", embeddedclusteroperator.Version),
111+
"local-artifact-mirror-image": defaults.LocalArtifactMirrorImage,
112+
}
113+
106114
meta := types.ReleaseMetadata{
107-
Versions: versions,
108-
K0sSHA: sha,
109-
K0sBinaryURL: defaults.K0sBinaryURL,
115+
Versions: versions,
116+
K0sSHA: sha,
117+
Artifacts: artifacts,
110118
}
111119

112120
chtconfig, repconfig, err := applier.GenerateHelmConfigs(
@@ -145,9 +153,6 @@ func gatherVersionMetadata() (*types.ReleaseMetadata, error) {
145153
return nil, fmt.Errorf("unable to get airgap images: %w", err)
146154
}
147155
meta.K0sImages = append(meta.K0sImages, additionalImages...)
148-
meta.Artifacts = map[string]string{
149-
"local-artifact-mirror-image": defaults.LocalArtifactMirrorImage,
150-
}
151156

152157
return &meta, nil
153158
}

‎e2e/version_test.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,23 @@ func TestVersion(t *testing.T) {
143143
}
144144
}
145145

146+
expectedArtifacts := []string{"kots", "operator", "local-artifact-mirror-image"}
147+
if len(parsed.Artifacts) != len(expectedArtifacts) {
148+
t.Log(output)
149+
t.Fatalf("found %d artifacts in metadata, expected %d", len(parsed.Artifacts), len(expectedArtifacts))
150+
}
151+
152+
for _, expectedName := range expectedArtifacts {
153+
if _, ok := parsed.Artifacts[expectedName]; !ok {
154+
t.Errorf("failed to find artifact %s in 'metadata' output", expectedName)
155+
failed = true
156+
}
157+
if len(parsed.Artifacts[expectedName]) == 0 {
158+
t.Errorf("artifact %s is empty in 'metadata' output", expectedName)
159+
failed = true
160+
}
161+
}
162+
146163
if failed {
147164
t.Log(output)
148165
t.FailNow()

‎go.mod

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,15 @@ require (
5757
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
5858
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
5959
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
60-
go.etcd.io/etcd/api/v3 v3.5.13 // indirect
61-
go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect
62-
go.etcd.io/etcd/client/v3 v3.5.13 // indirect
60+
go.etcd.io/etcd/api/v3 v3.5.14 // indirect
61+
go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
62+
go.etcd.io/etcd/client/v3 v3.5.14 // indirect
6363
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
6464
golang.org/x/sync v0.7.0 // indirect
6565
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
66-
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
67-
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
66+
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
6867
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
69-
google.golang.org/grpc v1.62.1 // indirect
68+
google.golang.org/grpc v1.64.0 // indirect
7069
helm.sh/helm/v3 v3.15.1 // indirect
7170
)
7271

‎go.sum

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -301,12 +301,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
301301
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
302302
github.com/zitadel/oidc/v2 v2.7.0 h1:IGX4EDk6tegTjUSsZDWeTfLseFU0BdJ/Glf1tgys2lU=
303303
github.com/zitadel/oidc/v2 v2.7.0/go.mod h1:zkUkVJS0sDVy9m0UA9RgO3f8i/C0rtjvXU36UJj7T+0=
304-
go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
305-
go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c=
306-
go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg=
307-
go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8=
308-
go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js=
309-
go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI=
304+
go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0=
305+
go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU=
306+
go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ=
307+
go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI=
308+
go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg=
309+
go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk=
310310
go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
311311
go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
312312
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -415,17 +415,15 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
415415
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
416416
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
417417
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
418-
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
419-
google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro=
420-
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU=
421-
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA=
418+
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
419+
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
422420
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
423421
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
424422
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
425423
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
426424
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
427-
google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
428-
google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
425+
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
426+
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
429427
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
430428
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
431429
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

‎pkg/addons/adminconsole/adminconsole.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ var (
4141
Version = "v0.0.0"
4242
ImageOverride = ""
4343
MigrationsImageOverride = ""
44+
KotsVersion = ""
4445
CounterRegex = regexp.MustCompile(`(\d+)/(\d+)`)
4546
Password = ""
4647
)

‎pkg/defaults/defaults.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@ var (
1616
KubectlVersion = "0.0.0"
1717
// provider holds a global reference to the default provider.
1818
provider *Provider
19-
// K0sBinaryURL holds an alternative URL from where to download the k0s
20-
// binary that has been embedded in this version of the binary. If this
21-
// is empty then it means we have shipped the official k0s binary. This
22-
// is set at compile time via ldflags.
23-
K0sBinaryURL = ""
2419
// LocalArtifactMirrorImage holds a reference to where the lam image for
2520
// this version of embedded-cluster is stored. Set at compile time.
2621
LocalArtifactMirrorImage = ""

‎scripts/cache-files.sh

Lines changed: 65 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22

3-
set -eo pipefail
3+
set -euo pipefail
44

55
function require() {
66
if [ -z "$2" ]; then
@@ -51,19 +51,78 @@ function k0sbin() {
5151
return 0
5252
fi
5353

54-
# if the override is set, the binary will have been added to the bucket through another process
54+
# if the override is set, we should download this binary and upload it to the bucket so as not to require end users hit the override url
5555
if [ -n "${k0s_override}" ] && [ "${k0s_override}" != '' ]; then
5656
echo "K0S_BINARY_SOURCE_OVERRIDE is set to '${k0s_override}', using that source"
57-
curl -L -o "${k0s_version}" "${k0s_override}"
57+
curl --fail-with-body -L -o "${k0s_version}" "${k0s_override}"
5858
else
5959
# download the k0s binary from official sources
60-
curl -L -o "${k0s_version}" "https://github.com/k0sproject/k0s/releases/download/${k0s_version}/k0s-${k0s_version}-amd64"
60+
echo "downloading k0s binary from https://github.com/k0sproject/k0s/releases/download/${k0s_version}/k0s-${k0s_version}-amd64"
61+
curl --fail-with-body -L -o "${k0s_version}" "https://github.com/k0sproject/k0s/releases/download/${k0s_version}/k0s-${k0s_version}-amd64"
6162
fi
6263

6364
# upload the binary to the bucket
6465
retry 3 aws s3 cp "${k0s_version}" "s3://${S3_BUCKET}/k0s-binaries/${k0s_version}"
6566
}
6667

68+
function operatorbin() {
69+
# first, figure out what version of operator is in the current build
70+
local operator_version=
71+
operator_version=$(awk '/^EMBEDDED_OPERATOR_CHART_VERSION/{print $3}' Makefile)
72+
73+
# check if the binary already exists in the bucket
74+
local operator_binary_exists=
75+
operator_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "operator-binaries/${operator_version}" || true)
76+
77+
# if the binary already exists, we don't need to upload it again
78+
if [ -n "${operator_binary_exists}" ]; then
79+
echo "operator binary ${operator_version} already exists in bucket ${S3_BUCKET}, skipping upload"
80+
return 0
81+
fi
82+
83+
# download the operator binary from github
84+
echo "downloading embedded cluster operator binary from https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
85+
curl --fail-with-body -L -o "${operator_version}" "https://github.com/replicatedhq/embedded-cluster-operator/releases/download/v${operator_version}/manager"
86+
87+
# upload the binary to the bucket
88+
retry 3 aws s3 cp "${operator_version}" "s3://${S3_BUCKET}/operator-binaries/${operator_version}"
89+
}
90+
91+
function kotsbin() {
92+
# first, figure out what version of kots is in the current build
93+
local kots_version=
94+
kots_version=$(awk '/^ADMIN_CONSOLE_CHART_VERSION/{print $3}' Makefile)
95+
kots_version=$(echo "${kots_version}" | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
96+
97+
local kots_override=
98+
kots_override=$(awk '/^KOTS_BINARY_URL_OVERRIDE/{gsub("\"", "", $3); print $3}' Makefile)
99+
100+
# check if the binary already exists in the bucket
101+
local kots_binary_exists=
102+
kots_binary_exists=$(aws s3api head-object --bucket "${S3_BUCKET}" --key "kots-binaries/${kots_version}" || true)
103+
104+
# if the binary already exists, we don't need to upload it again
105+
if [ -n "${kots_binary_exists}" ]; then
106+
echo "kots binary ${kots_version} already exists in bucket ${S3_BUCKET}, skipping upload"
107+
return 0
108+
fi
109+
110+
if [ -n "${kots_override}" ] && [ "${kots_override}" != '' ]; then
111+
echo "KOTS_BINARY_URL_OVERRIDE is set to '${kots_override}', using that source"
112+
curl --fail-with-body -L -o "kots_linux_amd64.tar.gz" "${kots_override}"
113+
else
114+
# download the kots binary from github
115+
echo "downloading kots binary from https://github.com/replicatedhq/kots/releases/download/v${kots_version}/kots_linux_amd64.tar.gz"
116+
curl --fail-with-body -L -o "kots_linux_amd64.tar.gz" "https://github.com/replicatedhq/kots/releases/download/v${kots_version}/kots_linux_amd64.tar.gz"
117+
fi
118+
119+
# decompress the bundle, as we only care about the binary and not the sbom/license/readme
120+
tar -xvf kots_linux_amd64.tar.gz
121+
122+
# upload the binary to the bucket
123+
retry 3 aws s3 cp "kots" "s3://${S3_BUCKET}/kots-binaries/${kots_version}"
124+
}
125+
67126
function metadata() {
68127
if [ -z "${EC_VERSION}" ]; then
69128
echo "EC_VERSION unset, not uploading metadata.json"
@@ -98,6 +157,8 @@ function embeddedcluster() {
98157
# the embedded cluster release does not exist for CI builds
99158
function main() {
100159
k0sbin
160+
operatorbin
161+
kotsbin
101162
metadata
102163
embeddedcluster
103164
}

0 commit comments

Comments
 (0)