Merge remote-tracking branch 'origin/main' into k0s-1-28

Author: emosbaugh

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,29 @@
+#### What this PR does / why we need it:
+<!--
+Describe the purpose of this change and the problem it solves.
+-->
+
+#### Which issue(s) this PR fixes:
+<!--
+Link to the Shortcut story or Github issue this PR fixes.
+-->
+
+#### Does this PR require a test?
+<!---
+If no, just write "NONE" below.
+-->
+
+#### Does this PR require a release note?
+<!--
+If no, just write "NONE" in the release-note block below.
+If yes, a release note is required:
+-->
+```release-note
+
+```
+
+#### Does this PR require documentation?
+<!--
+If no, just write "NONE" below.
+If yes, link to the related https://github.com/replicatedhq/replicated-docs documentation PR:
+-->

.github/dependabot.yml

@@ -16,6 +16,21 @@ updates:
         update-types:
           - "patch"
 
+  - package-ecosystem: "gomod"
+    directory: "/operator/"
+    open-pull-requests-limit: 25
+    schedule:
+      interval: "weekly"
+      day: "saturday"
+    labels:
+      - "dependencies"
+      - "go"
+      - "type::chore"
+    groups:
+      security:
+        update-types:
+          - "patch"
+
   - package-ecosystem: "github-actions"
     directory: "/"
     labels:

.github/workflows/ci.yaml

@@ -33,9 +33,6 @@ jobs:
             build/.melange-cache
           key: melange-cache
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Setup Melange
         uses: chainguard-dev/actions/setup-melange@main
 

.github/workflows/image-scan.yaml

@@ -2,7 +2,8 @@ name: Release
 on:
   push:
     tags:
-      - "*.*.*"
+      - '[0-9]+\.[0-9]+\.[0-9]+\+k8s-[0-9]+\.[0-9]+'
+      - '[0-9]+\.[0-9]+\.[0-9]+\+k8s-[0-9]+\.[0-9]+-.+'
 permissions:
   contents: write
 jobs:
@@ -21,17 +22,38 @@ jobs:
         id: get-tag
         run: |
           # remove the "refs/tags/" prefix to get the tag that was pushed
-          export RAW_TAG=${GITHUB_REF#refs/tags/}
+          export RAW_TAG=${{ github.ref_name }}
           # add a 'v' prefix to the tag if it doesn't already have one
           export V_TAG=$(echo "$RAW_TAG" | sed 's/^[^v]/v&/')
           # store the tag name in an output for later steps
           echo "tag-name=${V_TAG}" >> $GITHUB_OUTPUT
 
-  build-images:
+  buildtools:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Compile buildtools
+        run: |
+          make buildtools
+
+      - name: Upload buildtools artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: buildtools
+          path: output/bin/buildtools
+
+  publish-operator-image:
     runs-on: ubuntu-latest
     needs: [get-tag]
     outputs:
-      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
+      image: ${{ steps.operator-image.outputs.image }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -43,8 +65,55 @@ jobs:
             build/.melange-cache
           key: melange-cache
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Setup Melange
+        uses: chainguard-dev/actions/setup-melange@main
+
+      - name: Build and push operator image
+        id: operator-image
+        env:
+          REGISTRY: docker.io
+          USERNAME: ${{ secrets.DOCKERHUB_USER }}
+          PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+        run: |
+          make -C operator apko apko-login build-and-push-operator-image \
+            PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
+          echo "image=$(cat operator/build/image)" >> $GITHUB_OUTPUT
+
+  publish-operator-chart:
+    runs-on: ubuntu-latest
+    needs: [get-tag, publish-operator-image]
+    outputs:
+      chart: ${{ steps.operator-chart.outputs.chart }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build and push operator chart
+        id: operator-chart
+        env:
+          HELM_USER: ${{secrets.REPLICATED_LIBRARY_SERVICE_ACCOUNT}}
+          HELM_PASS: ${{secrets.REPLICATED_LIBRARY_SERVICE_ACCOUNT}}
+          HELM_REGISTRY: registry.replicated.com
+        run: |
+          make -C operator build-chart \
+            PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
+          echo "chart=$(cat operator/build/chart)" >> $GITHUB_OUTPUT
+
+  publish-images:
+    runs-on: ubuntu-latest
+    needs: [get-tag]
+    outputs:
+      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Cache Melange
+        uses: actions/cache@v4
+        with:
+          path: |
+            build/.melange-cache
+          key: melange-cache
 
       - name: Setup Melange
         uses: chainguard-dev/actions/setup-melange@main
@@ -62,29 +131,41 @@ jobs:
 
   release:
     runs-on: ubuntu-latest
-    needs: [get-tag, build-images]
+    needs: [get-tag, buildtools, publish-images, publish-operator-image, publish-operator-chart]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Set up Go
+      - name: Setup Go
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Download buildtools artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: buildtools
+          path: output/bin
+
+      - name: Update embedded-cluster-operator metadata.yaml
+        env:
+          IMAGES_REGISTRY_SERVER: ttl.sh
+          OPERATOR_CHART: ${{ needs.publish-operator-chart.outputs.chart }}
+          OPERATOR_IMAGE: ${{ needs.publish-operator-image.outputs.image }}
+        run: |
+          ./scripts/ci-update-operator-metadata.sh
 
       - name: Build linux-amd64
         run: |
           make embedded-cluster-linux-amd64 \
             VERSION=${{ needs.get-tag.outputs.tag-name }} \
-            LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.build-images.outputs.local-artifact-mirror }}
+            LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-images.outputs.local-artifact-mirror }}
           tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
 
       - name: Output Metadata
         run: |
           ./output/bin/embedded-cluster version metadata > metadata.json
+
       - name: Cache Staging Files
         env:
           S3_BUCKET: "tf-staging-embedded-cluster-bin"
@@ -93,6 +174,9 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
+          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-operator-image.outputs.image }}
+          export OPERATOR_VERSION=$(echo "${{ needs.get-tag.outputs.tag-name }}" | sed 's/^v//')
+
           ./scripts/cache-files.sh
 
       - name: Cache Prod Files
@@ -103,13 +187,16 @@ jobs:
           AWS_REGION: "us-east-1"
         run: |
           export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
+          export OPERATOR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-operator-image.outputs.image }}
+          export OPERATOR_VERSION=$(echo "${{ needs.get-tag.outputs.tag-name }}" | sed 's/^v//')
+
           ./scripts/cache-files.sh
 
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          prerelease: false
+          prerelease: true
           files: |
             *.tgz
             metadata.json

.github/workflows/release-prod.yaml

@@ -1,8 +1,8 @@
 SHELL := /bin/bash
 
-include chainguard.mk
+include common.mk
 
-VERSION ?= $(shell git describe --tags --dirty)
+VERSION ?= $(shell git describe --tags --dirty --match='[0-9]*.[0-9]*.[0-9]*')
 CURRENT_USER := $(if $(GITHUB_USER),$(GITHUB_USER),$(shell id -u -n))
 UNAME := $(shell uname)
 ARCH := $(shell uname -m)
@@ -11,14 +11,12 @@ ADMIN_CONSOLE_CHART_REPO_OVERRIDE =
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE =
-EMBEDDED_OPERATOR_IMAGE_OVERRIDE =
-EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE =
 K0S_VERSION = v1.28.11+k0s.0
 K0S_GO_VERSION = v1.28.11+k0s.0
 PREVIOUS_K0S_VERSION ?= v1.28.10+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE =
 PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE =
-TROUBLESHOOT_VERSION = v0.97.0
+TROUBLESHOOT_VERSION = v0.100.0
 KOTS_VERSION = v$(shell awk '/^version/{print $$2}' pkg/addons/adminconsole/static/metadata.yaml | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
 KOTS_BINARY_URL_OVERRIDE =
 # TODO: move this to a manifest file
@@ -34,8 +32,8 @@ LD_FLAGS = \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleChartRepoOverride=$(ADMIN_CONSOLE_CHART_REPO_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleImageOverride=$(ADMIN_CONSOLE_IMAGE_OVERRIDE) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleMigrationsImageOverride=$(ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE) \
-	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleKurlProxyImageOverride=$(ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE) \
-	-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.EmbeddedOperatorImageOverride=$(EMBEDDED_OPERATOR_IMAGE_OVERRIDE)
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.AdminConsoleKurlProxyImageOverride=$(ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE)
+DISABLE_FIO_BUILD ?= 0
 
 export PATH := $(shell pwd)/bin:$(PATH)
 
@@ -48,25 +46,25 @@ default: build-ttl.sh
 pkg/goods/bins/k0s: Makefile
 	mkdir -p pkg/goods/bins
 	if [ "$(K0S_BINARY_SOURCE_OVERRIDE)" != "" ]; then \
-	    curl -fL -o pkg/goods/bins/k0s "$(K0S_BINARY_SOURCE_OVERRIDE)" ; \
+	    curl --retry 5 --retry-all-errors -fL -o pkg/goods/bins/k0s "$(K0S_BINARY_SOURCE_OVERRIDE)" ; \
 	else \
-	    curl -fL -o pkg/goods/bins/k0s "https://github.com/k0sproject/k0s/releases/download/$(K0S_VERSION)/k0s-$(K0S_VERSION)-amd64" ; \
+	    curl --retry 5 --retry-all-errors -fL -o pkg/goods/bins/k0s "https://github.com/k0sproject/k0s/releases/download/$(K0S_VERSION)/k0s-$(K0S_VERSION)-amd64" ; \
 	fi
 	chmod +x pkg/goods/bins/k0s
 	touch pkg/goods/bins/k0s
 
 pkg/goods/bins/kubectl-support_bundle: Makefile
 	mkdir -p pkg/goods/bins
 	mkdir -p output/tmp/support-bundle
-	curl -fL -o output/tmp/support-bundle/support-bundle.tar.gz https://github.com/replicatedhq/troubleshoot/releases/download/$(TROUBLESHOOT_VERSION)/support-bundle_linux_amd64.tar.gz
+	curl --retry 5 --retry-all-errors -fL -o output/tmp/support-bundle/support-bundle.tar.gz https://github.com/replicatedhq/troubleshoot/releases/download/$(TROUBLESHOOT_VERSION)/support-bundle_linux_amd64.tar.gz
 	tar -xzf output/tmp/support-bundle/support-bundle.tar.gz -C output/tmp/support-bundle
 	mv output/tmp/support-bundle/support-bundle pkg/goods/bins/kubectl-support_bundle
 	touch pkg/goods/bins/kubectl-support_bundle
 
 pkg/goods/bins/kubectl-preflight: Makefile
 	mkdir -p pkg/goods/bins
 	mkdir -p output/tmp/preflight
-	curl -fL -o output/tmp/preflight/preflight.tar.gz https://github.com/replicatedhq/troubleshoot/releases/download/$(TROUBLESHOOT_VERSION)/preflight_linux_amd64.tar.gz
+	curl --retry 5 --retry-all-errors -fL -o output/tmp/preflight/preflight.tar.gz https://github.com/replicatedhq/troubleshoot/releases/download/$(TROUBLESHOOT_VERSION)/preflight_linux_amd64.tar.gz
 	tar -xzf output/tmp/preflight/preflight.tar.gz -C output/tmp/preflight
 	mv output/tmp/preflight/preflight pkg/goods/bins/kubectl-preflight
 	touch pkg/goods/bins/kubectl-preflight
@@ -76,13 +74,23 @@ pkg/goods/bins/local-artifact-mirror: Makefile
 	$(MAKE) -C local-artifact-mirror build GOOS=linux GOARCH=amd64
 	cp local-artifact-mirror/bin/local-artifact-mirror-$(GOOS)-$(GOARCH) pkg/goods/bins/local-artifact-mirror
 
+pkg/goods/bins/fio: PLATFORM = linux/amd64
+pkg/goods/bins/fio: Makefile
+ifneq ($(DISABLE_FIO_BUILD),1)
+	mkdir -p pkg/goods/bins
+	docker build -t fio --build-arg PLATFORM=$(PLATFORM) fio
+	docker rm -f fio && docker run --name fio fio
+	docker cp fio:/output/fio pkg/goods/bins/fio
+	touch pkg/goods/bins/fio
+endif
+
 pkg/goods/internal/bins/kubectl-kots: Makefile
 	mkdir -p pkg/goods/internal/bins
 	mkdir -p output/tmp/kots
 	if [ "$(KOTS_BINARY_URL_OVERRIDE)" != "" ]; then \
-	    curl -fL -o output/tmp/kots/kots.tar.gz "$(KOTS_BINARY_URL_OVERRIDE)" ; \
+	    curl --retry 5 --retry-all-errors -fL -o output/tmp/kots/kots.tar.gz "$(KOTS_BINARY_URL_OVERRIDE)" ; \
 	else \
-	    curl -fL -o output/tmp/kots/kots.tar.gz https://github.com/replicatedhq/kots/releases/download/$(KOTS_VERSION)/kots_linux_amd64.tar.gz ; \
+	    curl --retry 5 --retry-all-errors -fL -o output/tmp/kots/kots.tar.gz https://github.com/replicatedhq/kots/releases/download/$(KOTS_VERSION)/kots_linux_amd64.tar.gz ; \
 	fi
 	tar -xzf output/tmp/kots/kots.tar.gz -C output/tmp/kots
 	mv output/tmp/kots/kots pkg/goods/internal/bins/kubectl-kots
@@ -110,6 +118,7 @@ static: pkg/goods/bins/k0s \
 	pkg/goods/bins/kubectl-preflight \
 	pkg/goods/bins/kubectl-support_bundle \
 	pkg/goods/bins/local-artifact-mirror \
+	pkg/goods/bins/fio \
 	pkg/goods/internal/bins/kubectl-kots
 
 .PHONY: embedded-cluster-linux-amd64
@@ -138,6 +147,7 @@ unit-tests:
 	mkdir -p pkg/goods/bins pkg/goods/internal/bins
 	touch pkg/goods/bins/BUILD pkg/goods/internal/bins/BUILD # compilation will fail if no files are present
 	go test -v ./pkg/... ./cmd/...
+	$(MAKE) -C operator test
 
 .PHONY: vet
 vet: static
@@ -190,7 +200,6 @@ buildtools:
 	go build -o ./output/bin/buildtools ./cmd/buildtools
 
 .PHONY: cache-files
-cache-files: export EMBEDDED_OPERATOR_BINARY_URL_OVERRIDE
 cache-files:
 	./scripts/cache-files.sh
 

Makefile

@@ -49,12 +49,25 @@ var updateOperatorAddonCommand = &cli.Command{
 		}
 		nextChartVersion = strings.TrimPrefix(nextChartVersion, "v")
 
-		upstream := "registry.replicated.com/library/embedded-cluster-operator"
-		withproto := fmt.Sprintf("oci://proxy.replicated.com/anonymous/%s", upstream)
+		chartURL := os.Getenv("INPUT_OPERATOR_CHART_URL")
+		if chartURL != "" {
+			logrus.Infof("using input override from INPUT_OPERATOR_CHART_URL: %s", chartURL)
+		} else {
+			chartURL = "registry.replicated.com/library/embedded-cluster-operator"
+			chartURL = fmt.Sprintf("oci://proxy.replicated.com/anonymous/%s", chartURL)
+		}
 
-		logrus.Infof("updating embedded cluster operator images")
+		imageOverride := os.Getenv("INPUT_OPERATOR_IMAGE")
+		if imageOverride != "" {
+			logrus.Infof("using input override from INPUT_OPERATOR_IMAGE: %s", imageOverride)
+
+			operatorImageComponents[imageOverride] = addonComponent{
+				name:             "embedded-cluster-operator",
+				useUpstreamImage: true,
+			}
+		}
 
-		err := updateOperatorAddonImages(c.Context, withproto, nextChartVersion)
+		err := updateOperatorAddonImages(c.Context, chartURL, nextChartVersion)
 		if err != nil {
 			return fmt.Errorf("failed to update embedded cluster operator images: %w", err)
 		}