|
| 1 | +#!/usr/bin/env python3 |
| 2 | +import argparse |
| 3 | +import contextlib |
| 4 | +import functools |
| 5 | +import re |
| 6 | +import subprocess |
| 7 | +import sys |
| 8 | +import time |
| 9 | +import typing |
| 10 | +import unittest |
| 11 | +import unittest.mock |
| 12 | + |
| 13 | +"""Runs the make commands used to deploy, test, and undeploy image in Kubernetes |
| 14 | +
|
| 15 | +The make commands this runs are intended to reproduce the commands we define in our OpenShift CI config at |
| 16 | +https://github.com/openshift/release/blob/master/ci-operator/config/opendatahub-io/notebooks/opendatahub-io-notebooks-main.yaml#L1485 |
| 17 | +""" |
| 18 | + |
| 19 | + |
| 20 | +class Args(argparse.Namespace): |
| 21 | + """Type annotation to have autocompletion for args""" |
| 22 | + target: str |
| 23 | + |
| 24 | + |
| 25 | +def main() -> None: |
| 26 | + parser = argparse.ArgumentParser("make_test.py") |
| 27 | + parser.add_argument("--target", type=str) |
| 28 | + args = typing.cast(Args, parser.parse_args()) |
| 29 | + |
| 30 | + run_tests(args.target) |
| 31 | + |
| 32 | + |
| 33 | +def run_tests(target: str) -> None: |
| 34 | + prefix = target.translate(str.maketrans(".", "-")) |
| 35 | + # this is a pod name in statefulset, some tests deploy individual unmanaged pods, though |
| 36 | + pod = prefix + "-notebook-0" # `$(kubectl get statefulset -o name | head -n 1)` would work too |
| 37 | + namespace = "ns-" + prefix |
| 38 | + |
| 39 | + if target.startswith("runtime-"): |
| 40 | + deploy = "deploy9" |
| 41 | + deploy_target = target.replace("runtime-", "runtimes-") |
| 42 | + elif target.startswith("intel-runtime-"): |
| 43 | + deploy = "deploy9" |
| 44 | + deploy_target = target.replace("intel-runtime-", "intel-runtimes-") |
| 45 | + elif target.startswith("rocm-runtime-"): |
| 46 | + deploy = "deploy9" |
| 47 | + deploy_target = target.replace("rocm-runtime-", "runtimes-rocm-") |
| 48 | + elif target.startswith("rocm-jupyter-"): |
| 49 | + deploy = "deploy9" |
| 50 | + deploy_target = target.replace("rocm-jupyter-", "jupyter-rocm-") |
| 51 | + elif target.startswith("cuda-rstudio-"): |
| 52 | + deploy = "deploy" |
| 53 | + os = re.match(r"^cuda-rstudio-([^-]+-).*", target) |
| 54 | + deploy_target = os.group(1) + target.removeprefix("cuda-") |
| 55 | + elif target.startswith("rstudio-"): |
| 56 | + deploy = "deploy" |
| 57 | + os = re.match(r"^rstudio-([^-]+-).*", target) |
| 58 | + deploy_target = os.group(1) + target |
| 59 | + else: |
| 60 | + deploy = "deploy9" |
| 61 | + deploy_target = target |
| 62 | + |
| 63 | + check_call(f"kubectl create namespace {namespace}", shell=True) |
| 64 | + check_call(f"kubectl config set-context --current --namespace={namespace}", shell=True) |
| 65 | + check_call(f"kubectl label namespace {namespace} fake-scc=fake-restricted-v2", shell=True) |
| 66 | + |
| 67 | + # wait for service account to be created, otherwise pod is refused to be created |
| 68 | + # $ bin/kubectl apply -k runtimes/minimal/ubi9-python-3.9/kustomize/base |
| 69 | + # configmap/runtime-req-config-9hhb2bhhmd created |
| 70 | + # Error from server (Forbidden): error when creating "runtimes/minimal/ubi9-python-3.9/kustomize/base": pods "runtime-pod" is forbidden: error looking up service account ns-runtime-minimal-ubi9-python-3-9/default: serviceaccount "default" not found |
| 71 | + # See https://github.com/kubernetes/kubernetes/issues/66689 |
| 72 | + check_call(f"timeout 10s bash -c 'until kubectl get serviceaccount/default; do sleep 1; done'", shell=True) |
| 73 | + |
| 74 | + check_call(f"make {deploy}-{deploy_target}", shell=True) |
| 75 | + wait_for_stability(pod) |
| 76 | + |
| 77 | + try: |
| 78 | + if target.startswith("runtime-") or target.startswith("intel-runtime-"): |
| 79 | + check_call(f"make validate-runtime-image image={target}", shell=True) |
| 80 | + elif target.startswith("rocm-runtime-"): |
| 81 | + check_call(f"make validate-runtime-image image={target |
| 82 | + .replace("rocm-runtime-", "runtime-rocm-")}", shell=True) |
| 83 | + elif target.startswith("rstudio-") or target.startswith("cuda-rstudio-"): |
| 84 | + check_call(f"make validate-rstudio-image image={target}", shell=True) |
| 85 | + elif target.startswith("codeserver-"): |
| 86 | + check_call(f"make validate-codeserver-image image={target}", shell=True) |
| 87 | + elif target.startswith("rocm-jupyter"): |
| 88 | + check_call(f"make test-{target |
| 89 | + .replace("rocm-jupyter-", "jupyter-rocm-")}", shell=True) |
| 90 | + else: |
| 91 | + check_call(f"make test-{target}", shell=True) |
| 92 | + finally: |
| 93 | + # dump a lot of info to the GHA logs |
| 94 | + with gha_log_group("pod and statefulset info"): |
| 95 | + call(f"kubectl get statefulsets", shell=True) |
| 96 | + call(f"kubectl describe statefulsets", shell=True) |
| 97 | + call(f"kubectl get pods", shell=True) |
| 98 | + call(f"kubectl describe pods", shell=True) |
| 99 | + # describe does not show everything about the pod |
| 100 | + call(f"kubectl get pods -o yaml", shell=True) |
| 101 | + |
| 102 | + with gha_log_group("kubernetes namespace events"): |
| 103 | + # events aren't all that useful, but it can tell what was happening in the current namespace |
| 104 | + call(f"kubectl get events", shell=True) |
| 105 | + |
| 106 | + with gha_log_group("previous pod logs"): |
| 107 | + # relevant if the pod is crashlooping, this shows the final lines |
| 108 | + # use the negative label selector as a trick to match all pods (as we don't have any pods with nosuchlabel) |
| 109 | + call(f"kubectl logs --selector=nosuchlabel!=nosuchvalue --all-pods --timestamps --previous", shell=True) |
| 110 | + with gha_log_group("current pod logs"): |
| 111 | + # regular logs from a running (or finished) pod |
| 112 | + call(f"kubectl logs --selector=nosuchlabel!=nosuchvalue --all-pods --timestamps", shell=True) |
| 113 | + |
| 114 | + check_call(f"make un{deploy}-{deploy_target}", shell=True) |
| 115 | + |
| 116 | + print(f"[INFO] Finished testing {target}") |
| 117 | + |
| 118 | + |
| 119 | +@functools.wraps(subprocess.check_call) |
| 120 | +def check_call(*args, **kwargs) -> int: |
| 121 | + return execute(subprocess.check_call, args, kwargs) |
| 122 | + |
| 123 | + |
| 124 | +@functools.wraps(subprocess.call) |
| 125 | +def call(*args, **kwargs) -> int: |
| 126 | + return execute(subprocess.call, args, kwargs) |
| 127 | + |
| 128 | + |
| 129 | +def execute(executor: typing.Callable, args: tuple, kwargs: dict) -> int: |
| 130 | + print(f"[INFO] Running command {args, kwargs}") |
| 131 | + sys.stdout.flush() |
| 132 | + result = executor(*args, **kwargs) |
| 133 | + print(f"\tDONE running command {args, kwargs}") |
| 134 | + sys.stdout.flush() |
| 135 | + return result |
| 136 | + |
| 137 | + |
| 138 | +# TODO(jdanek) this is a dumb impl, needs to be improved |
| 139 | +def wait_for_stability(pod: str) -> None: |
| 140 | + """Waits for the pod to be stable. Often I'm seeing that the probes initially fail. |
| 141 | + > error: Internal error occurred: error executing command in container: container is not created or running |
| 142 | + > error: unable to upgrade connection: container not found ("notebook") |
| 143 | + """ |
| 144 | + timeout = 100 |
| 145 | + for _ in range(3): |
| 146 | + call( |
| 147 | + f"timeout {timeout}s bash -c 'until kubectl wait --for=condition=Ready pods --all --timeout 5s; do sleep 1; done'", shell=True) |
| 148 | + timeout = 50 |
| 149 | + time.sleep(3) |
| 150 | + |
| 151 | + |
| 152 | +# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines |
| 153 | +@contextlib.contextmanager |
| 154 | +def gha_log_group(title): |
| 155 | + """Prints the starting and ending magic strings for GitHub Actions line group in log.""" |
| 156 | + print(f"::group::{title}", file=sys.stdout) |
| 157 | + sys.stdout.flush() |
| 158 | + try: |
| 159 | + yield |
| 160 | + finally: |
| 161 | + print("::endgroup::", file=sys.stdout) |
| 162 | + sys.stdout.flush() |
| 163 | + |
| 164 | + |
| 165 | +# https://docs.python.org/3/library/unittest.mock-examples.html#patch-decorators |
| 166 | +@unittest.mock.patch("time.sleep", unittest.mock.Mock()) |
| 167 | +class TestMakeTest(unittest.TestCase): |
| 168 | + @unittest.mock.patch("make_test.execute") |
| 169 | + def test_make_commands_jupyter(self, mock_execute: unittest.mock.Mock) -> None: |
| 170 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 171 | + run_tests("jupyter-minimal-ubi9-python-3.11") |
| 172 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 173 | + assert "make deploy9-jupyter-minimal-ubi9-python-3.11" in commands |
| 174 | + assert "make test-jupyter-minimal-ubi9-python-3.11" in commands |
| 175 | + assert "make undeploy9-jupyter-minimal-ubi9-python-3.11" in commands |
| 176 | + |
| 177 | + @unittest.mock.patch("make_test.execute") |
| 178 | + def test_make_commands_jupyter_rocm(self, mock_execute: unittest.mock.Mock) -> None: |
| 179 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 180 | + run_tests("rocm-jupyter-tensorflow-ubi9-python-3.11") |
| 181 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 182 | + assert "make deploy9-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands |
| 183 | + assert "make test-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands |
| 184 | + assert "make undeploy9-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands |
| 185 | + |
| 186 | + @unittest.mock.patch("make_test.execute") |
| 187 | + def test_make_commands_codeserver(self, mock_execute: unittest.mock.Mock) -> None: |
| 188 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 189 | + run_tests("codeserver-ubi9-python-3.11") |
| 190 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 191 | + assert "make deploy9-codeserver-ubi9-python-3.11" in commands |
| 192 | + assert "make validate-codeserver-image image=codeserver-ubi9-python-3.11" in commands |
| 193 | + assert "make undeploy9-codeserver-ubi9-python-3.11" in commands |
| 194 | + |
| 195 | + @unittest.mock.patch("make_test.execute") |
| 196 | + def test_make_commands_rstudio(self, mock_execute: unittest.mock.Mock) -> None: |
| 197 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 198 | + run_tests("rstudio-c9s-python-3.11") |
| 199 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 200 | + assert "make deploy-c9s-rstudio-c9s-python-3.11" in commands |
| 201 | + assert "make validate-rstudio-image image=rstudio-c9s-python-3.11" in commands |
| 202 | + assert "make undeploy-c9s-rstudio-c9s-python-3.11" in commands |
| 203 | + |
| 204 | + @unittest.mock.patch("make_test.execute") |
| 205 | + def test_make_commands_cuda_rstudio(self, mock_execute: unittest.mock.Mock) -> None: |
| 206 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 207 | + run_tests("cuda-rstudio-c9s-python-3.11") |
| 208 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 209 | + assert "make deploy-c9s-rstudio-c9s-python-3.11" in commands |
| 210 | + assert "make validate-rstudio-image image=cuda-rstudio-c9s-python-3.11" in commands |
| 211 | + assert "make undeploy-c9s-rstudio-c9s-python-3.11" in commands |
| 212 | + |
| 213 | + @unittest.mock.patch("make_test.execute") |
| 214 | + def test_make_commands_runtime(self, mock_execute: unittest.mock.Mock) -> None: |
| 215 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 216 | + run_tests("runtime-datascience-ubi9-python-3.11") |
| 217 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 218 | + assert "make deploy9-runtimes-datascience-ubi9-python-3.11" in commands |
| 219 | + assert "make validate-runtime-image image=runtime-datascience-ubi9-python-3.11" in commands |
| 220 | + assert "make undeploy9-runtimes-datascience-ubi9-python-3.11" in commands |
| 221 | + |
| 222 | + @unittest.mock.patch("make_test.execute") |
| 223 | + def test_make_commands_intel_runtime(self, mock_execute: unittest.mock.Mock) -> None: |
| 224 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 225 | + run_tests("intel-runtime-ml-ubi9-python-3.11") |
| 226 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 227 | + assert "make deploy9-intel-runtimes-ml-ubi9-python-3.11" in commands |
| 228 | + assert "make validate-runtime-image image=intel-runtime-ml-ubi9-python-3.11" in commands |
| 229 | + assert "make undeploy9-intel-runtimes-ml-ubi9-python-3.11" in commands |
| 230 | + |
| 231 | + @unittest.mock.patch("make_test.execute") |
| 232 | + def test_make_commands_rocm_runtime(self, mock_execute: unittest.mock.Mock) -> None: |
| 233 | + """Compares the commands with what we had in the openshift/release yaml""" |
| 234 | + run_tests("rocm-runtime-pytorch-ubi9-python-3.11") |
| 235 | + commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list] |
| 236 | + assert "make deploy9-runtimes-rocm-pytorch-ubi9-python-3.11" in commands |
| 237 | + assert "make validate-runtime-image image=runtime-rocm-pytorch-ubi9-python-3.11" in commands |
| 238 | + assert "make undeploy9-runtimes-rocm-pytorch-ubi9-python-3.11" in commands |
| 239 | + |
| 240 | + |
| 241 | +if __name__ == "__main__": |
| 242 | + main() |
0 commit comments