Skip to content
This repository was archived by the owner on Dec 9, 2022. It is now read-only.

Commit fdf63ca

Browse files
author
awmatheson
committed
Revert "Update Jupyter Blog Posts"
This reverts commit abceb01.
1 parent a91b934 commit fdf63ca

File tree

102 files changed

+10018
-5137
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

102 files changed

+10018
-5137
lines changed

‎.dockerignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
*.hdf5
2+
.*
3+
metadata.json
4+
dask-worker-space
5+

‎.github/workflows/chatops.yaml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: ChatOps
2+
on: [issue_comment]
3+
4+
jobs:
5+
full-test-run:
6+
if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/run-full-test') && (github.actor == 'hamelsmu' || github.actor == 'hohsiangwu' || github.actor == 'inc0' || github.actor == 'awmatheson')
7+
runs-on: ubuntu-latest
8+
steps:
9+
- uses: machine-learning-apps/actions-chatops@master
10+
with:
11+
APP_PEM: ${{ secrets.APP_PEM }}
12+
APP_ID: ${{ secrets.APP_ID }}
13+
TRIGGER_PHRASE: "/run-full-test"
14+
INDICATOR_LABEL: "Full Test Pending"
15+
env:
16+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17+
deploy:
18+
if: github.event.issue.pull_request != null && contains(github.event.comment.body, '/deploy') && (github.actor == 'hamelsmu' || github.actor == 'hohsiangwu' || github.actor == 'inc0')
19+
runs-on: ubuntu-latest
20+
steps:
21+
22+
- uses: machine-learning-apps/actions-chatops@master
23+
id: deploy_chatops
24+
with:
25+
APP_PEM: ${{ secrets.APP_PEM }}
26+
APP_ID: ${{ secrets.APP_ID }}
27+
TRIGGER_PHRASE: "/deploy"
28+
env:
29+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30+
31+
- name: validate deploy argument
32+
uses: machine-learning-apps/wandb-action@master
33+
if: steps.deploy_chatops.outputs.BOOL_TRIGGERED == 'true'
34+
id: wandb
35+
with:
36+
PROJECT_NAME: ${{ format('{0}/{1}', secrets.WANDB_ENTITY, secrets.WANDB_PROJECT) }}
37+
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
38+
RUN_ID: ${{ steps.deploy_chatops.outputs.TRAILING_TOKEN }}
39+
DEBUG: 'true'
40+
41+
- name: Copy Repository Contents
42+
uses: actions/checkout@master
43+
44+
- name: create deployment
45+
if: steps.wandb.outputs.BOOL_SINGLE_RUN
46+
run: |
47+
pip3 install requests
48+
python3 action_files/create_deployment.py
49+
# curl -d "{\"ref\": \"$BRANCH_NAME\", \"payload\": {\"wandb_run_id\": \"$RUN_ID\"}}" -H "Authorization: token $TOKEN" https://api.github.com/repos/$GITHUB_REPOSITORY/deployments
50+
env:
51+
RUN_ID: ${{ steps.deploy_chatops.outputs.TRAILING_TOKEN }}
52+
BRANCH_NAME: ${{ steps.deploy_chatops.outputs.BRANCH_NAME }}
53+
TOKEN: ${{ steps.deploy_chatops.outputs.APP_INSTALLATION_TOKEN }}
54+
55+

‎.github/workflows/copy_to_pages.yaml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Project Data to Pages
2+
on: [push]
3+
# push:
4+
# branches:
5+
# - master
6+
# paths:
7+
# - 'project/model_card.md'
8+
# - 'project/data_dictionary.md'
9+
# - 'project/model_events.csv'
10+
11+
jobs:
12+
copy-build-deploy:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@master
16+
17+
- name: Copy Repository Contents
18+
run: |
19+
chmod -R 777 docs/docs
20+
./action_files/copy_files.sh
21+
./action_files/push_to_repo.sh
22+
23+
- name: setup directories for Jekyll build
24+
run: |
25+
cd docs
26+
rm -rf _site
27+
sudo chmod -R 777 .
28+
29+
# - name: Setup tmate session
30+
# uses: mxschmitt/action-tmate@v1
31+
32+
- name: Jekyll build
33+
run: |
34+
docker run -v /home/runner/work/actions-ml-cicd/actions-ml-cicd/docs:/workspace -w /workspace jekyll/jekyll jekyll build
35+
36+
- name: Deploy
37+
uses: peaceiris/actions-gh-pages@v2
38+
env:
39+
ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEPLOY_KEY }}
40+
PUBLISH_DIR: ./docs/_site
41+
PUBLISH_BRANCH: master

‎.github/workflows/deploy.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Google Cloud Function Deploy
2+
3+
on: deployment
4+
5+
jobs:
6+
deploy:
7+
if: github.actor == 'pr-chatops[bot]'
8+
runs-on: ubuntu-latest
9+
steps:
10+
11+
- uses: actions/checkout@v1
12+
13+
- uses: exelban/gcloud@master
14+
env:
15+
PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }}
16+
APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
17+
with:
18+
args: functions deploy predict --runtime python37 --trigger-http --source=$GITHUB_WORKSPACE/src/deploy --set-env-vars WANDB_API_KEY=${{ secrets.WANDB_API_KEY }},WANDB_ENTITY=${{ secrets.WANDB_ENTITY }},WANDB_PROJECT=${{ secrets.WANDB_PROJECT }},WANDB_RUN_ID=${{ github.event.deployment.payload.wandb_run_id }}
19+
- run: |
20+
curl -d '{"state": "failure"}' -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/"$GITHUB_REPOSITORY"/deployments/"$DEPLOYMENT_ID"/statuses
21+
if: failure()
22+
env:
23+
GITHUB_TOKEN: ${{ github.token }}
24+
DEPLOYMENT_ID: ${{ github.event.deployment.id }}
25+
- run: |
26+
curl -d "{\"state\": \"success\", \"description\":\"Deployed run id: $WANDB_RUN_ID\", \"log_url\": \"https://console.cloud.google.com/functions/details/us-central1/predict?project=$GOOGLE_PROJECT_ID\"}" -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.ant-man-preview+json" https://api.github.com/repos/"$GITHUB_REPOSITORY"/deployments/"$DEPLOYMENT_ID"/statuses
27+
if: success()
28+
env:
29+
GITHUB_TOKEN: ${{ github.token }}
30+
DEPLOYMENT_ID: ${{ github.event.deployment.id }}
31+
PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }}
32+
WANDB_RUN_ID: ${{ github.event.deployment.payload.wandb_run_id }}

‎.github/workflows/ml-cicd.yaml

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: ML Workflow Via Actions
2+
on:
3+
pull_request:
4+
types: [labeled]
5+
6+
jobs:
7+
8+
ml-workflow:
9+
if: github.actor == 'pr-chatops[bot]'
10+
runs-on: ubuntu-latest
11+
steps:
12+
13+
- name: Copy Repository Contents
14+
uses: actions/checkout@master
15+
16+
- name: Trigger on PR Label
17+
id: label
18+
run: python action_files/validate_payload.py
19+
env:
20+
TRIGGER_LABEL_NAME: "Full Test Pending"
21+
22+
- name: Publish CPU Image to DockerHub
23+
if: steps.label.outputs.TRIGGERED == 'true'
24+
run: |
25+
cd $GITHUB_WORKSPACE
26+
echo ${PASSWORD} | docker login -u $USERNAME --password-stdin
27+
BASE_NAME="$USERNAME/ml-cicd"
28+
IMAGE_NAME="$BASE_NAME:$GITHUB_SHA"
29+
docker pull $BASE_NAME || true
30+
docker build --cache-from $BASE_NAME -t $IMAGE_NAME -t $BASE_NAME -f docker/Dockerfile .
31+
docker push $IMAGE_NAME
32+
docker push $BASE_NAME
33+
env:
34+
USERNAME: ${{ secrets.DOCKER_USERNAME }}
35+
PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
36+
37+
- name: Publish GPU Image to DockerHub
38+
if: steps.label.outputs.TRIGGERED == 'true'
39+
run: |
40+
cd $GITHUB_WORKSPACE
41+
echo ${PASSWORD} | docker login -u $USERNAME --password-stdin
42+
BASE_NAME="$USERNAME/ml-cicd-gpu"
43+
IMAGE_NAME="$BASE_NAME:$GITHUB_SHA"
44+
docker pull $BASE_NAME || true
45+
docker build --cache-from $BASE_NAME -t $IMAGE_NAME -t $BASE_NAME -f docker/Dockerfile .
46+
docker push $IMAGE_NAME
47+
docker push $BASE_NAME
48+
env:
49+
USERNAME: ${{ secrets.DOCKER_USERNAME }}
50+
PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
51+
52+
- name: Build Argo Param File
53+
if: steps.label.outputs.TRIGGERED == 'true'
54+
run: |
55+
cd $GITHUB_WORKSPACE
56+
echo "GITHUB-SHA: $SHA" >> pipelines/params.yaml
57+
echo "PR-NUMBER: $PR_NUMBER" >> pipelines/params.yaml
58+
echo "REPO: $GITHUB_REPOSITORY" >> pipelines/params.yaml
59+
env:
60+
SHA: ${{ steps.label.outputs.HEAD_SHA }}
61+
PR_NUMBER: ${{ steps.label.outputs.ISSUE_NUMBER }}
62+
63+
- name: Submit Argo Deployment
64+
if: steps.label.outputs.TRIGGERED == 'true'
65+
id: argo
66+
uses: machine-learning-apps/gke-argo@master
67+
with:
68+
ARGO_URL: ${{ secrets.ARGO_URI }}
69+
APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
70+
PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }}
71+
LOCATION_ZONE: "us-west1-a"
72+
CLUSTER_NAME: "github-actions-demo"
73+
WORKFLOW_YAML_PATH: "pipelines/workflow.yaml" # the argo workflow file relative to the repo's root.
74+
PARAMETER_FILE_PATH: "pipelines/params.yaml" # optional parameter file.
75+
SHA: ${{ steps.label.outputs.HEAD_SHA }}
76+
77+
- name: Create Check
78+
if: steps.label.outputs.TRIGGERED == 'true'
79+
run: |
80+
python3 action_files/create_check_run.py "in_progress" "Argo-Workflow" "ML Workflow For SHA $SHA" "Links [Argo Workflow]($ARGO_LINK) | [W&B Project](https://app.wandb.ai/$WANDB_ENTITY/$WANDB_PROJECT) | Docker: [hamelsmu/ml-cicd](https://cloud.docker.com/u/hamelsmu/repository/docker/hamelsmu/ml-cicd), [hamelsmu/ml-cicd-gpu](https://cloud.docker.com/u/hamelsmu/repository/docker/hamelsmu/ml-cicd-gpu)" " "
81+
env:
82+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
83+
ARGO_LINK: ${{ steps.argo.outputs.WORKFLOW_URL }}
84+
WANDB_ENTITY: ${{ secrets.WANDB_ENTITY }}
85+
WANDB_PROJECT: ${{ secrets.WANDB_PROJECT }}
86+
SHA: ${{ steps.label.outputs.HEAD_SHA }}
87+
88+
- name: Comment With Metadata
89+
if: steps.label.outputs.TRIGGERED == 'true'
90+
run: |
91+
bash action_files/pr_comment.sh "### [ML Workflow]($ARGO_LINK) For SHA $SHA has been instantiated.\n\nThe following Docker images were built and tagged with the SHA: \n - [hamelsmu/ml-cicd](https://cloud.docker.com/u/hamelsmu/repository/docker/hamelsmu/ml-cicd)\n - [hamelsmu/ml-cicd-gpu](https://cloud.docker.com/u/hamelsmu/repository/docker/hamelsmu/ml-cicd-gpu)\n\nCheck run \`Argo-Workflow\` created, with status pending completion of the workflow."
92+
env:
93+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
94+
ARGO_LINK: ${{ steps.argo.outputs.WORKFLOW_URL }}
95+
WANDB_ENTITY: ${{ secrets.WANDB_ENTITY }}
96+
WANDB_PROJECT: ${{ secrets.WANDB_PROJECT }}
97+
ISSUE_NUMBER: ${{ steps.label.outputs.ISSUE_NUMBER }}
98+
SHA: ${{ steps.label.outputs.HEAD_SHA }}

‎.github/workflows/nb2docs.yaml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: Convert Notebooks To Docs
2+
on:
3+
push:
4+
branches:
5+
- master
6+
paths:
7+
- 'notebooks/*.ipynb'
8+
9+
jobs:
10+
convert-notebook:
11+
if: contains(join(github.event.commits.*.message), '/sync')
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Copy Repository Contents
15+
uses: actions/checkout@master
16+
with:
17+
persist-credentials: false
18+
19+
- uses: fastai/fastpages@master
20+
with:
21+
BOOL_SAVE_MARKDOWN: true
22+
SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
23+
FORMAT: 'notebook'
24+
25+
- name: setup directories for Jekyll build
26+
run: |
27+
rm -rf _site
28+
sudo chmod -R 777 .
29+
- name: Jekyll build
30+
uses: docker://jekyll/jekyll
31+
with:
32+
args: jekyll build
33+
34+
- name: Deploy
35+
uses: peaceiris/actions-gh-pages@v2
36+
env:
37+
ACTIONS_DEPLOY_KEY: ${{ secrets.ACTIONS_DEPLOY_KEY }}
38+
PUBLISH_DIR: ./_site
39+
keep_files: true
40+

‎.github/workflows/repo-dispatch.yaml

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Receive Repository Dispatch Payload
2+
on: repository_dispatch
3+
4+
jobs:
5+
process-repository-dispatch:
6+
if: github.actor == 'pr-chatops[bot]' || github.actor == 'hamelsmu' || github.actor == 'hohsiangwu'
7+
runs-on: ubuntu-latest
8+
9+
steps:
10+
- name: Copy Repository Contents
11+
uses: actions/checkout@master
12+
13+
- name: handle repo dispatch payload
14+
id: rd
15+
run: |
16+
python3 action_files/repo_dispatch_handler.py
17+
env:
18+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
19+
20+
- name: Get Runs Using SHA
21+
uses: machine-learning-apps/wandb-action@master
22+
if: steps.rd.outputs.CONCLUSION == 'success'
23+
id: wandb
24+
with:
25+
PROJECT_NAME: ${{ format('{0}/{1}', secrets.WANDB_ENTITY, secrets.WANDB_PROJECT) }}
26+
FILTER_GITHUB_SHA: ${{ steps.rd.outputs.SHA }}
27+
BASELINE_TAGS: "['baseline', 'reference']"
28+
DISPLAY_METRICS: "['accuracy', 'loss', 'best_val_acc', 'best_val_loss', '_runtime']"
29+
WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
30+
DEBUG: 'true'
31+
32+
- name: Upload csv
33+
if: steps.wandb.outputs.BOOL_COMPLETE == 'true' && steps.rd.outputs.CONCLUSION == 'success'
34+
uses: actions/upload-artifact@v1
35+
with:
36+
name: wandb_report
37+
path: wandb_report.csv
38+
39+
- name: process_data
40+
if: steps.wandb.outputs.BOOL_COMPLETE == 'true' && steps.rd.outputs.CONCLUSION == 'success'
41+
run: |
42+
cd $GITHUB_WORKSPACE
43+
sudo apt-get install python3-setuptools
44+
pip3 install wheel
45+
pip3 install pandas==0.25.1 tabulate==0.8.5 requests==2.22.0
46+
python3 action_files/wandb_results.py
47+
env:
48+
PR_NUM: ${{ steps.rd.outputs.ISSUE_NUMBER }}
49+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
50+
51+
- name: update status of check run
52+
run: |
53+
cd $GITHUB_WORKSPACE
54+
python3 action_files/clear_check_run.py
55+
env:
56+
WORKFLOW_CONCLUSION: ${{ steps.rd.outputs.CONCLUSION }}
57+
SHA: ${{ steps.rd.outputs.SHA }}
58+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/see-payload.yaml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: see-payload
2+
on: [gollum]
3+
4+
jobs:
5+
6+
see-payload:
7+
runs-on: ubuntu-latest
8+
steps:
9+
10+
- name: write payload
11+
run: |
12+
cat $GITHUB_EVENT_PATH > payload.json
13+
14+
- name: Upload payload
15+
uses: actions/upload-artifact@v1
16+
with:
17+
name: payload
18+
path: payload.json

‎.github/workflows/see_token.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: Get token for prototyping
2+
on: [gollum]
3+
4+
jobs:
5+
test:
6+
name: see-token
7+
runs-on: ubuntu-latest
8+
9+
steps:
10+
- name: Write token
11+
run: |
12+
echo "Hello" > tok.txt
13+
env:
14+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
15+
- name: Upload payload
16+
uses: actions/upload-artifact@v1
17+
with:
18+
name: token
19+
path: tok.txt

‎404.html

-1
This file was deleted.

0 commit comments

Comments
 (0)