Skip to content

Commit cfae89c

Browse files
xuzhao9facebook-github-bot
xuzhao9
authored and
facebook-github-bot
committed
Added torchao nightly workflow (#2273)
Summary: X-link: pytorch/pytorch#128152 Add torchao benchmark workflow, upload the artifacts to GHA. Pull Request resolved: #2273 Test Plan: ``` python run_benchmark.py torchao --ci ``` Reviewed By: jerryzh168 Differential Revision: D58140479 Pulled By: xuzhao9 fbshipit-source-id: b274edb417f880df9b149bd5afc1acbe95737433
1 parent f7b4bcc commit cfae89c

File tree

11 files changed

+441
-110
lines changed

11 files changed

+441
-110
lines changed

‎.github/workflows/torchao.yml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
name: Torchao nightly workflow (A100)
2+
on:
3+
workflow_dispatch:
4+
5+
6+
jobs:
7+
run-benchmark:
8+
environment: docker-s3-upload
9+
env:
10+
BASE_CONDA_ENV: "torchbench"
11+
CONDA_ENV: "torchao-nightly"
12+
PLATFORM_NAME: "gcp_a100"
13+
SETUP_SCRIPT: "/workspace/setup_instance.sh"
14+
TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN }}
15+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
16+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
17+
IS_GHA: 1
18+
BUILD_ENVIRONMENT: benchmark-nightly
19+
if: ${{ github.repository_owner == 'pytorch' }}
20+
runs-on: [a100-runner]
21+
timeout-minutes: 1440 # 24 hours
22+
steps:
23+
- name: Checkout TorchBench
24+
uses: actions/checkout@v3
25+
with:
26+
path: benchmark
27+
- name: Tune Nvidia GPU
28+
run: |
29+
sudo nvidia-smi -pm 1
30+
sudo nvidia-smi -ac 1215,1410
31+
nvidia-smi
32+
sudo ldconfig
33+
- name: Clone and setup conda env
34+
run: |
35+
CONDA_ENV=${BASE_CONDA_ENV} . "${SETUP_SCRIPT}"
36+
conda create --name "${CONDA_ENV}" --clone "${BASE_CONDA_ENV}"
37+
- name: Run the torchao userbenchmark
38+
env:
39+
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
40+
WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
41+
run: |
42+
. "${SETUP_SCRIPT}"
43+
set -x
44+
# remove old results if exists
45+
if [ -d benchmark-output ]; then rm -Rf benchmark-output; fi
46+
pushd benchmark
47+
if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi
48+
# Install torchao
49+
echo "Installing torchao"
50+
pip uninstall -y torchao
51+
python install.py --userbenchmark torchao
52+
echo "Running the torchao userbenchmark"
53+
python run_benchmark.py torchao --ci --dashboard
54+
- name: Copy the benchmark logs to benchmark-output
55+
if: always()
56+
run: |
57+
pushd benchmark
58+
cp -r ./.userbenchmark/torchao ../benchmark-output
59+
- name: Upload result to GH Actions Artifact
60+
uses: actions/upload-artifact@v3
61+
if: always()
62+
with:
63+
name: Torchao nightly result
64+
path: benchmark-output/
65+
- name: Copy artifact and upload to scribe and Amazon S3
66+
env:
67+
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
68+
WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
69+
run: |
70+
. "${SETUP_SCRIPT}"
71+
pushd benchmark
72+
# Upload the result json to Amazon S3
73+
python ./scripts/userbenchmark/upload_s3_csv.py --s3-prefix torchbench-csv --userbenchmark torchao \
74+
--upload-path ../benchmark-output --match-filename "^torchao_.*\.csv"
75+
- name: Clean up Conda env
76+
if: always()
77+
run: |
78+
. "${SETUP_SCRIPT}"
79+
conda deactivate && conda deactivate
80+
conda remove -n "${CONDA_ENV}" --all

‎scripts/userbenchmark/upload_s3_csv.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import argparse
2+
import sys
3+
import os
4+
import re
5+
from pathlib import Path
6+
from datetime import datetime
7+
8+
REPO_ROOT = Path(__file__).parent.parent.parent.resolve()
9+
10+
class add_path:
11+
def __init__(self, path):
12+
self.path = path
13+
14+
def __enter__(self):
15+
sys.path.insert(0, self.path)
16+
17+
def __exit__(self, exc_type, exc_value, traceback):
18+
try:
19+
sys.path.remove(self.path)
20+
except ValueError:
21+
pass
22+
23+
24+
with add_path(str(REPO_ROOT)):
25+
from utils.s3_utils import (
26+
S3Client,
27+
USERBENCHMARK_S3_BUCKET,
28+
)
29+
30+
31+
def upload_s3(s3_object: str,
32+
ub_name: str,
33+
workflow_run_id: str,
34+
workflow_run_attempt: str,
35+
file_path: Path,
36+
dryrun: bool):
37+
"""S3 path:
38+
s3://ossci-metrics/<s3_object>/<ub_name>/<workflow_run_id>/<workflow_run_attempt>/file_name
39+
"""
40+
s3client = S3Client(USERBENCHMARK_S3_BUCKET, s3_object)
41+
prefix = f"{ub_name}/{workflow_run_id}/{workflow_run_attempt}"
42+
print(f"Uploading to prefix: {prefix}")
43+
if not dryrun:
44+
s3client.upload_file(prefix=prefix, file_path=file_path)
45+
46+
47+
def _get_files_to_upload(file_path: str, match_filename: str):
48+
filename_regex = re.compile(match_filename)
49+
return [ file_name for file_name in os.listdir(file_path) if filename_regex.match(file_name) ]
50+
51+
if __name__ == "__main__":
52+
parser = argparse.ArgumentParser(description=__doc__)
53+
parser.add_argument(
54+
"--s3-prefix",
55+
required=True,
56+
help="S3 path prefix",
57+
)
58+
parser.add_argument(
59+
"--userbenchmark",
60+
required=True,
61+
help="Name of the userbenchmark.",
62+
)
63+
parser.add_argument(
64+
"--upload-path",
65+
required=True,
66+
help="Local directory contains files to upload.",
67+
)
68+
parser.add_argument(
69+
"--match-filename",
70+
required=True,
71+
help="Filename regex matched to upload.",
72+
)
73+
parser.add_argument(
74+
"--dryrun",
75+
action="store_true",
76+
help="Dryrun the upload",
77+
)
78+
args = parser.parse_args()
79+
80+
files_to_upload = _get_files_to_upload(args.upload_path, args.match_filename)
81+
workflow_run_id = os.environ.get("WORKFLOW_RUN_ID", 0)
82+
workflow_run_attempt = os.environ.get("WORKFLOW_RUN_ATTEMPT", 0)
83+
84+
for file in files_to_upload:
85+
file_path = Path(args.upload_path).joinpath(file)
86+
upload_s3(s3_object=args.s3_prefix,
87+
ub_name=args.userbenchmark,
88+
workflow_run_id=workflow_run_id,
89+
workflow_run_attempt=workflow_run_attempt,
90+
file_path=file_path,
91+
dryrun=args.dryrun)

‎torchbenchmark/util/framework/huggingface/extended_configs.py

Lines changed: 0 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,8 @@
11
# Extended huggingface model configs from Dynamobench
22
import importlib
33
import logging
4-
import os
5-
from typing import List
64

75
import torch
8-
from torchbenchmark import REPO_PATH
9-
10-
DYNAMOBENCH_PATH = REPO_PATH.joinpath("userbenchmark", "dynamo", "dynamobench")
11-
# These models contain the models present in huggingface_models_list. It is a
12-
# combination of models supported by HF Fx parser and some manually supplied
13-
# models. For these models, we already know the largest batch size that can fit
14-
# on A100 GPUs - 40 GB.
15-
BATCH_SIZE_KNOWN_MODELS = dict()
16-
17-
# Get the list of models and their batch sizes
18-
# Only load the extended models in OSS
19-
if hasattr(torch.version, "git_version"):
20-
MODELS_FILENAME = os.path.join(DYNAMOBENCH_PATH, "huggingface_models_list.txt")
21-
else:
22-
from libfb.py import parutil
23-
MODELS_FILENAME = parutil.get_file_path("caffe2/benchmarks/dynamo/huggingface_models_list.txt")
24-
assert os.path.exists(MODELS_FILENAME)
25-
with open(MODELS_FILENAME, "r") as fh:
26-
lines = fh.readlines()
27-
lines = [line.rstrip() for line in lines]
28-
for line in lines:
29-
model_name, batch_size = line.split(",")
30-
batch_size = int(batch_size)
31-
BATCH_SIZE_KNOWN_MODELS[model_name] = batch_size
32-
assert len(BATCH_SIZE_KNOWN_MODELS)
33-
34-
35-
def is_extended_huggingface_models(model_name: str) -> bool:
36-
return model_name in BATCH_SIZE_KNOWN_MODELS
37-
38-
39-
def list_extended_huggingface_models() -> List[str]:
40-
return list(BATCH_SIZE_KNOWN_MODELS.keys())
41-
426

437
imports = [
448
"AlbertForPreTraining",
@@ -161,61 +125,6 @@ def list_extended_huggingface_models() -> List[str]:
161125
"tinynet_a",
162126
}
163127

164-
# TODO - Fails even after fake tensors
165-
BATCH_SIZE_DIVISORS = {
166-
"AlbertForMaskedLM": 2,
167-
"AlbertForQuestionAnswering": 2,
168-
"AllenaiLongformerBase": 2,
169-
"BartForCausalLM": 2,
170-
"BartForConditionalGeneration": 2,
171-
"BertForMaskedLM": 2,
172-
"BertForQuestionAnswering": 2,
173-
"BlenderbotForCausalLM": 8,
174-
# "BlenderbotForConditionalGeneration" : 16,
175-
"BlenderbotSmallForCausalLM": 4,
176-
"BlenderbotSmallForConditionalGeneration": 2,
177-
"CamemBert": 2,
178-
"DebertaForMaskedLM": 4,
179-
"DebertaForQuestionAnswering": 2,
180-
"DebertaV2ForMaskedLM": 4,
181-
"DebertaV2ForQuestionAnswering": 8,
182-
"DistilBertForMaskedLM": 2,
183-
"DistilBertForQuestionAnswering": 2,
184-
"DistillGPT2": 2,
185-
"ElectraForCausalLM": 2,
186-
"ElectraForQuestionAnswering": 2,
187-
"GPT2ForSequenceClassification": 2,
188-
# "GPTJForCausalLM" : 2,
189-
# "GPTJForQuestionAnswering" : 2,
190-
# "GPTNeoForCausalLM" : 32,
191-
# "GPTNeoForSequenceClassification" : 2,
192-
"GoogleFnet": 2,
193-
"LayoutLMForMaskedLM": 2,
194-
"LayoutLMForSequenceClassification": 2,
195-
"M2M100ForConditionalGeneration": 4,
196-
"MBartForCausalLM": 2,
197-
"MBartForConditionalGeneration": 2,
198-
"MT5ForConditionalGeneration": 2,
199-
"MegatronBertForCausalLM": 4,
200-
"MegatronBertForQuestionAnswering": 2,
201-
"MobileBertForMaskedLM": 2,
202-
"MobileBertForQuestionAnswering": 2,
203-
"OPTForCausalLM": 2,
204-
"PLBartForCausalLM": 2,
205-
"PLBartForConditionalGeneration": 2,
206-
"PegasusForCausalLM": 4,
207-
"PegasusForConditionalGeneration": 2,
208-
"RobertaForCausalLM": 2,
209-
"RobertaForQuestionAnswering": 2,
210-
"Speech2Text2ForCausalLM": 4,
211-
"T5ForConditionalGeneration": 2,
212-
"T5Small": 2,
213-
"TrOCRForCausalLM": 2,
214-
"XGLMForCausalLM": 4,
215-
"XLNetLMHeadModel": 2,
216-
"YituTechConvBert": 2,
217-
}
218-
219128
try:
220129
EXTRA_MODELS = {
221130
"AllenaiLongformerBase": (

‎torchbenchmark/util/framework/huggingface/list_extended_configs.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import torch
2+
import os
3+
from torchbenchmark import REPO_PATH
4+
5+
from typing import List
6+
7+
DYNAMOBENCH_PATH = REPO_PATH.joinpath("userbenchmark", "dynamo", "dynamobench")
8+
9+
# These models contain the models present in huggingface_models_list. It is a
10+
# combination of models supported by HF Fx parser and some manually supplied
11+
# models. For these models, we already know the largest batch size that can fit
12+
# on A100 GPUs - 40 GB.
13+
BATCH_SIZE_KNOWN_MODELS = dict()
14+
15+
# Get the list of models and their batch sizes
16+
# Only load the extended models in OSS
17+
if hasattr(torch.version, "git_version"):
18+
MODELS_FILENAME = os.path.join(DYNAMOBENCH_PATH, "huggingface_models_list.txt")
19+
else:
20+
from libfb.py import parutil
21+
MODELS_FILENAME = parutil.get_file_path("caffe2/benchmarks/dynamo/huggingface_models_list.txt")
22+
assert os.path.exists(MODELS_FILENAME)
23+
with open(MODELS_FILENAME, "r") as fh:
24+
lines = fh.readlines()
25+
lines = [line.rstrip() for line in lines]
26+
for line in lines:
27+
model_name, batch_size = line.split(",")
28+
batch_size = int(batch_size)
29+
BATCH_SIZE_KNOWN_MODELS[model_name] = batch_size
30+
assert len(BATCH_SIZE_KNOWN_MODELS)
31+
32+
def is_extended_huggingface_models(model_name: str) -> bool:
33+
return model_name in BATCH_SIZE_KNOWN_MODELS
34+
35+
def list_extended_huggingface_models() -> List[str]:
36+
return list(BATCH_SIZE_KNOWN_MODELS.keys())
37+
38+
# TODO - Fails even after fake tensors
39+
BATCH_SIZE_DIVISORS = {
40+
"AlbertForMaskedLM": 2,
41+
"AlbertForQuestionAnswering": 2,
42+
"AllenaiLongformerBase": 2,
43+
"BartForCausalLM": 2,
44+
"BartForConditionalGeneration": 2,
45+
"BertForMaskedLM": 2,
46+
"BertForQuestionAnswering": 2,
47+
"BlenderbotForCausalLM": 8,
48+
# "BlenderbotForConditionalGeneration" : 16,
49+
"BlenderbotSmallForCausalLM": 4,
50+
"BlenderbotSmallForConditionalGeneration": 2,
51+
"CamemBert": 2,
52+
"DebertaForMaskedLM": 4,
53+
"DebertaForQuestionAnswering": 2,
54+
"DebertaV2ForMaskedLM": 4,
55+
"DebertaV2ForQuestionAnswering": 8,
56+
"DistilBertForMaskedLM": 2,
57+
"DistilBertForQuestionAnswering": 2,
58+
"DistillGPT2": 2,
59+
"ElectraForCausalLM": 2,
60+
"ElectraForQuestionAnswering": 2,
61+
"GPT2ForSequenceClassification": 2,
62+
# "GPTJForCausalLM" : 2,
63+
# "GPTJForQuestionAnswering" : 2,
64+
# "GPTNeoForCausalLM" : 32,
65+
# "GPTNeoForSequenceClassification" : 2,
66+
"GoogleFnet": 2,
67+
"LayoutLMForMaskedLM": 2,
68+
"LayoutLMForSequenceClassification": 2,
69+
"M2M100ForConditionalGeneration": 4,
70+
"MBartForCausalLM": 2,
71+
"MBartForConditionalGeneration": 2,
72+
"MT5ForConditionalGeneration": 2,
73+
"MegatronBertForCausalLM": 4,
74+
"MegatronBertForQuestionAnswering": 2,
75+
"MobileBertForMaskedLM": 2,
76+
"MobileBertForQuestionAnswering": 2,
77+
"OPTForCausalLM": 2,
78+
"PLBartForCausalLM": 2,
79+
"PLBartForConditionalGeneration": 2,
80+
"PegasusForCausalLM": 4,
81+
"PegasusForConditionalGeneration": 2,
82+
"RobertaForCausalLM": 2,
83+
"RobertaForQuestionAnswering": 2,
84+
"Speech2Text2ForCausalLM": 4,
85+
"T5ForConditionalGeneration": 2,
86+
"T5Small": 2,
87+
"TrOCRForCausalLM": 2,
88+
"XGLMForCausalLM": 4,
89+
"XLNetLMHeadModel": 2,
90+
"YituTechConvBert": 2,
91+
}

‎torchbenchmark/util/framework/huggingface/model_factory.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from transformers import GenerationConfig
1313

1414
from .basic_configs import is_basic_huggingface_models
15-
from .extended_configs import (
15+
from .list_extended_configs import (
1616
BATCH_SIZE_DIVISORS,
1717
BATCH_SIZE_KNOWN_MODELS,
1818
is_extended_huggingface_models,

‎userbenchmark/dynamo/dynamobench/common.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3974,9 +3974,12 @@ def run(runner, args, original_dir=None):
39743974
assert "cuda" in args.devices, "Quantization requires CUDA device."
39753975
assert args.bfloat16, "Quantization requires dtype bfloat16."
39763976
try:
3977-
from .torchao_backend import setup_baseline, torchao_optimize_ctx
3978-
except ImportError:
39793977
from torchao_backend import setup_baseline, torchao_optimize_ctx
3978+
except ImportError:
3979+
from userbenchmark.dynamo.dynamobench.torchao_backend import (
3980+
setup_baseline,
3981+
torchao_optimize_ctx,
3982+
)
39803983

39813984
setup_baseline()
39823985
baseline_ctx = functools.partial(

0 commit comments

Comments
 (0)