Skip to content

Commit 6e2e5e5

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #474 from atheo89/RHOAIENG-2928
Update digest updater workflow
2 parents c852d47 + fa5836c commit 6e2e5e5

File tree

2 files changed

+216
-39
lines changed

2 files changed

+216
-39
lines changed

‎.github/workflows/notebooks-digest-updater-upstream.yaml

Lines changed: 83 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -51,53 +51,75 @@ jobs:
5151
git config --global user.email "github-actions[bot]@users.noreply.github.com"
5252
git config --global user.name "GitHub Actions"
5353
54-
# Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2024a
55-
- name: Checkout upstream notebooks repo
56-
uses: actions/checkout@v4
57-
with:
58-
repository: opendatahub-io/notebooks.git
59-
ref: ${{ env.RELEASE_VERSION_N }}
60-
61-
- name: Retrieve latest weekly commit hash from the release branch
54+
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
55+
- name: Retrive latest commit hash from the release branch
6256
id: hash-n
6357
shell: bash
6458
run: |
65-
echo "HASH_N=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
59+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
60+
echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
6661
6762
# Checkout the release branch to apply the updates
6863
- name: Checkout release branch
6964
uses: actions/checkout@v4
7065
with:
7166
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
7267

73-
- name: Fetch digest, and update the params.env file
68+
- name: Update the param.env file
7469
run: |
7570
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
76-
IMAGES=("odh-minimal-notebook-image-n" "odh-minimal-gpu-notebook-image-n" "odh-pytorch-gpu-notebook-image-n" "odh-generic-data-science-notebook-image-n" "odh-tensorflow-gpu-notebook-image-n" "odh-trustyai-notebook-image-n")
71+
IMAGES=("odh-minimal-notebook-image-n"
72+
"odh-minimal-gpu-notebook-image-n"
73+
"odh-pytorch-gpu-notebook-image-n"
74+
"odh-generic-data-science-notebook-image-n"
75+
"odh-tensorflow-gpu-notebook-image-n"
76+
"odh-trustyai-notebook-image-n"
77+
"odh-codeserver-notebook-n"
78+
"odh-rstudio-notebook-n"
79+
"odh-rstudio-gpu-notebook-n")
80+
7781
for ((i=0;i<${#IMAGES[@]};++i)); do
7882
image=${IMAGES[$i]}
79-
echo $image
83+
echo "CHECKING: " $image
8084
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
8185
registry=$(echo $img | cut -d '@' -f1)
8286
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
8387
regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
8488
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
8589
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
8690
output=$registry@$digest
87-
echo $output
91+
echo "NEW:" $output
8892
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
8993
done
90-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
94+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
95+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
96+
else
97+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
98+
fi
9199
92-
- name: Fetch digest, and update the commit.env file
100+
- name: Update the commit.env file
93101
run: |
94102
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
95-
COMMIT=("odh-minimal-notebook-image-commit-n" "odh-minimal-gpu-notebook-image-commit-n" "odh-pytorch-gpu-notebook-image-commit-n" "odh-generic-data-science-notebook-image-commit-n" "odh-tensorflow-gpu-notebook-image-commit-n" "odh-trustyai-notebook-image-commit-n")
103+
COMMIT=("odh-minimal-notebook-image-commit-n"
104+
"odh-minimal-gpu-notebook-image-commit-n"
105+
"odh-pytorch-gpu-notebook-image-commit-n"
106+
"odh-generic-data-science-notebook-image-commit-n"
107+
"odh-tensorflow-gpu-notebook-image-commit-n"
108+
"odh-trustyai-notebook-image-commit-n"
109+
"odh-codeserver-notebook-n"
110+
"odh-rstudio-notebook-n"
111+
"odh-rstudio-gpu-notebook-n")
112+
96113
for val in "${COMMIT[@]}"; do
97114
echo $val
98115
sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
99116
done
100-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
117+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
118+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
119+
else
120+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
121+
fi
122+
101123
update-n-1-version:
102124
needs: [initialize, update-n-version]
103125
runs-on: ubuntu-latest
@@ -109,54 +131,74 @@ jobs:
109131
git config --global user.email "github-actions[bot]@users.noreply.github.com"
110132
git config --global user.name "GitHub Actions"
111133
112-
# Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023b
113-
- name: Checkout upstream notebooks repo
114-
uses: actions/checkout@v4
115-
with:
116-
repository: opendatahub-io/notebooks.git
117-
ref: ${{ env.RELEASE_VERSION_N_1 }}
118-
119-
- name: Retrieve latest weekly commit hash from the release branch
134+
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
135+
- name: Retrive latest commit hash from the release branch
120136
id: hash-n-1
121137
shell: bash
122138
run: |
123-
echo "HASH_N_1=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
139+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N_1&per_page=1)
140+
echo "HASH_N_1=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
124141
125142
# Checkout the release branch to apply the updates
126143
- name: Checkout release branch
127144
uses: actions/checkout@v4
128145
with:
129146
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
130147

131-
- name: Fetch digest, and update the params.env file
148+
- name: Update the param.env file
132149
run: |
133150
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
134-
IMAGES=("odh-minimal-notebook-image-n-1" "odh-minimal-gpu-notebook-image-n-1" "odh-pytorch-gpu-notebook-image-n-1" "odh-generic-data-science-notebook-image-n-1" "odh-tensorflow-gpu-notebook-image-n-1" "odh-trustyai-notebook-image-n-1")
151+
IMAGES=("odh-minimal-notebook-image-n-1"
152+
"odh-minimal-gpu-notebook-image-n-1"
153+
"odh-pytorch-gpu-notebook-image-n-1"
154+
"odh-generic-data-science-notebook-image-n-1"
155+
"odh-tensorflow-gpu-notebook-image-n-1"
156+
"odh-trustyai-notebook-image-n-1"
157+
"odh-codeserver-notebook-n-1"
158+
"odh-rstudio-notebook-n-1"
159+
"odh-rstudio-gpu-notebook-n-1")
160+
135161
for ((i=0;i<${#IMAGES[@]};++i)); do
136162
image=${IMAGES[$i]}
137-
echo $image
163+
echo "CHECKING: " $image
138164
img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
139165
registry=$(echo $img | cut -d '@' -f1)
140166
src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
141167
regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
142168
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
143169
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
144170
output=$registry@$digest
145-
echo $output
171+
echo "NEW:" $output
146172
sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
147173
done
148-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
149-
150-
- name: Fetch digest, and update the commit.env file
174+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
175+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
176+
else
177+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
178+
fi
179+
- name: Update the commit.env file
151180
run: |
152181
echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
153-
COMMIT=("odh-minimal-notebook-image-commit-n-1" "odh-minimal-gpu-notebook-image-commit-n-1" "odh-pytorch-gpu-notebook-image-commit-n-1" "odh-generic-data-science-notebook-image-commit-n-1" "odh-tensorflow-gpu-notebook-image-commit-n-1" "odh-trustyai-notebook-image-commit-n-1")
182+
COMMIT=("odh-minimal-notebook-image-commit-n-1"
183+
"odh-minimal-gpu-notebook-image-commit-n-1"
184+
"odh-pytorch-gpu-notebook-image-commit-n-1"
185+
"odh-generic-data-science-notebook-image-commit-n-1"
186+
"odh-tensorflow-gpu-notebook-image-commit-n-1"
187+
"odh-trustyai-notebook-image-commit-n-1"
188+
"odh-codeserver-notebook-n-1"
189+
"odh-rstudio-notebook-n-1"
190+
"odh-rstudio-gpu-notebook-n-1")
191+
154192
for val in "${COMMIT[@]}"; do
155193
echo $val
156194
sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
157195
done
158-
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
159-
# Creates the Pull Request
196+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
197+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
198+
else
199+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
200+
fi
201+
160202
open-pull-request:
161203
needs: [update-n-version, update-n-1-version]
162204
runs-on: ubuntu-latest
@@ -173,11 +215,13 @@ jobs:
173215
destination_branch: ${{ env.BRANCH_NAME}}
174216
github_token: ${{ secrets.GITHUB_TOKEN }}
175217
pr_label: "automated pr"
176-
pr_title: "[Digest Updater Action] Update notebook's imageStreams image tag to digest format"
218+
pr_title: "[Digest Updater Action] Update Notebook Images"
177219
pr_body: |
178220
:rocket: This is an automated Pull Request.
179-
180-
This PR updates the `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
181221
Created by `/.github/workflows/notebooks-digest-updater-upstream.yaml`
182222
223+
This PR updates the following files:
224+
- `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
225+
- `manifests/base/commit.env` file with the latest commit (N & N-1).
226+
183227
:exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes

‎.github/workflows/runtimes-digest-updater-upstream.yaml

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
---
2+
# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths.
3+
name: Update runtime images
4+
on: # yamllint disable-line rule:truthy
5+
workflow_dispatch:
6+
inputs:
7+
branch:
8+
required: true
9+
description: "Provide the name of the branch you want to update ex main, vYYYYx etc: "
10+
# Put the scheduler on comment until automate the full release procedure
11+
# schedule:
12+
# - cron: "0 0 * * 5" #Scheduled every Friday
13+
env:
14+
DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }}
15+
BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
16+
RELEASE_VERSION_N: 2024a
17+
jobs:
18+
initialize:
19+
runs-on: ubuntu-latest
20+
permissions:
21+
contents: write
22+
steps:
23+
- name: Install Skopeo CLI
24+
shell: bash
25+
run: |
26+
sudo apt-get -y update
27+
sudo apt-get -y install skopeo
28+
29+
# Checkout the branch
30+
- name: Checkout branch
31+
uses: actions/checkout@v4
32+
with:
33+
ref: ${{ env.BRANCH_NAME }}
34+
35+
# Create a new branch
36+
- name: Create a new branch
37+
run: |
38+
echo ${{ env.DIGEST_UPDATER_BRANCH }}
39+
git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }}
40+
git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }}
41+
42+
update-runtimes:
43+
needs: [initialize]
44+
runs-on: ubuntu-latest
45+
permissions:
46+
contents: write
47+
steps:
48+
- name: Configure Git
49+
run: |
50+
git config --global user.email "github-actions[bot]@users.noreply.github.com"
51+
git config --global user.name "GitHub Actions"
52+
53+
# Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name
54+
- name: Retrive latest commit hash from the release branch
55+
id: hash-n
56+
shell: bash
57+
run: |
58+
PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1)
59+
echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT}
60+
61+
# Checkout the release branch to apply the updates
62+
- name: Checkout release branch
63+
uses: actions/checkout@v4
64+
with:
65+
ref: ${{ env.DIGEST_UPDATER_BRANCH }}
66+
67+
- name: Update Runtimes
68+
run: |
69+
echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
70+
PATHS=("jupyter/datascience/anaconda-python-3.8/runtime-images/datascience-ubi8-py38.json"
71+
"jupyter/datascience/anaconda-python-3.8/runtime-images/pytorch-ubi8-py38.json"
72+
"jupyter/datascience/anaconda-python-3.8/runtime-images/tensorflow-ubi8-py38.json"
73+
"jupyter/datascience/anaconda-python-3.8/runtime-images/ubi8-py38.json"
74+
"jupyter/datascience/ubi8-python-3.8/runtime-images/datascience-ubi8-py38.json"
75+
"jupyter/datascience/ubi8-python-3.8/runtime-images/pytorch-ubi8-py38.json"
76+
"jupyter/datascience/ubi8-python-3.8/runtime-images/tensorflow-ubi8-py38.json"
77+
"jupyter/datascience/ubi8-python-3.8/runtime-images/ubi8-py38.json"
78+
"jupyter/datascience/ubi9-python-3.9/runtime-images/datascience-ubi9-py39.json"
79+
"jupyter/datascience/ubi9-python-3.9/runtime-images/pytorch-ubi9-py39.json"
80+
"jupyter/datascience/ubi9-python-3.9/runtime-images/tensorflow-ubi9-py39.json"
81+
"jupyter/datascience/ubi9-python-3.9/runtime-images/ubi9-py39.json")
82+
83+
for ((i=0;i<${#PATHS[@]};++i)); do
84+
path=${PATHS[$i]}
85+
img=$(cat ${path} | jq -r '.metadata.image_name')
86+
name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#')
87+
py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]')
88+
# Handling specific cases
89+
if [[ $name == *tensorflow* ]]; then
90+
name="cuda-$name"
91+
elif [[ $name == ubi* ]]; then
92+
name="minimal-$name"
93+
fi
94+
registry=$(echo $img | cut -d '@' -f1)
95+
regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
96+
echo "CHECKING: " $regex
97+
latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
98+
digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
99+
output=$registry@$digest
100+
echo "NEW: " $output
101+
jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path"
102+
done
103+
if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
104+
git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add jupyter/datascience/* && git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
105+
else
106+
echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
107+
fi
108+
109+
open-pull-request:
110+
needs: [update-runtimes]
111+
runs-on: ubuntu-latest
112+
permissions:
113+
pull-requests: write
114+
steps:
115+
- name: Checkout repo
116+
uses: actions/checkout@v4
117+
118+
- name: pull-request
119+
uses: repo-sync/pull-request@v2
120+
with:
121+
source_branch: ${{ env.DIGEST_UPDATER_BRANCH }}
122+
destination_branch: ${{ env.BRANCH_NAME}}
123+
github_token: ${{ secrets.GITHUB_TOKEN }}
124+
pr_label: "automated pr"
125+
pr_title: "[Digest Updater Action] Update Runtimes Images"
126+
pr_body: |
127+
:rocket: This is an automated Pull Request.
128+
Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml`
129+
130+
This PR updates the following files:
131+
- All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths
132+
133+
:exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes

0 commit comments

Comments
 (0)