Skip to content

Commit 1db74ee

Browse files
committed
Fix pip install requirements
1 parent 9f80f30 commit 1db74ee

File tree

11 files changed

+44
-78
lines changed

11 files changed

+44
-78
lines changed

install.py

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,11 @@
66

77
from userbenchmark import list_userbenchmarks
88
from utils import get_pkg_versions, TORCH_DEPS, generate_pkg_constraints
9+
from utils.python_utils import pip_install_requirements
910

1011
REPO_ROOT = Path(__file__).parent
1112

1213

13-
def pip_install_requirements(requirements_txt="requirements.txt"):
14-
try:
15-
subprocess.run(
16-
[sys.executable, "-m", "pip", "install", "-q", "-r", requirements_txt],
17-
check=True,
18-
stdout=subprocess.PIPE,
19-
stderr=subprocess.STDOUT,
20-
)
21-
except subprocess.CalledProcessError as e:
22-
return (False, e.output)
23-
except Exception as e:
24-
return (False, e)
25-
return True, None
26-
27-
2814
if __name__ == "__main__":
2915
parser = argparse.ArgumentParser()
3016
parser.add_argument(
@@ -87,7 +73,7 @@ def pip_install_requirements(requirements_txt="requirements.txt"):
8773
)
8874
sys.exit(0)
8975

90-
success, errmsg = pip_install_requirements()
76+
success, errmsg = pip_install_requirements(continue_on_fail=True)
9177
if not success:
9278
print("Failed to install torchbenchmark requirements:")
9379
print(errmsg)

requirements.txt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,7 @@ transformers==4.38.1
1515
MonkeyType
1616
psutil
1717
pyyaml
18-
# We need to pin numpy version to the same as the torch testing environment
19-
# which still supports python 3.8
20-
numpy==1.21.2; python_version < '3.11'
21-
numpy==1.26.0; python_version >= '3.11'
18+
numpy
2219
opencv-python
2320
submitit
2421
pynvml

torchbenchmark/models/Background_Matting/install.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
1-
import subprocess
2-
import sys
3-
from utils import s3_utils
1+
from utils import s3_utils, python_utils
42

53
def pip_install_requirements():
6-
subprocess.check_call([sys.executable, '-m', 'pip',
7-
'install', '-q', '-r', 'requirements.txt'])
4+
python_utils.pip_install_requirements('requirements.txt')
85

96
if __name__ == '__main__':
107
pip_install_requirements()

torchbenchmark/models/Background_Matting/requirements.txt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
# We need to pin numpy version to the same as the torch testing environment
2-
# which still supports python 3.8
3-
numpy==1.21.2; python_version < '3.11'
4-
numpy==1.26.0; python_version >= '3.11'
1+
numpy
52
opencv-python
63
pandas
74
Pillow

torchbenchmark/models/dcgan/install.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,4 @@
1-
import subprocess
2-
import sys
3-
4-
5-
def pip_install_requirements():
6-
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
1+
from utils.python_utils import pip_install_requirements
72

83
if __name__ == '__main__':
94
pip_install_requirements()
Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,4 @@
1-
import subprocess
2-
import sys
3-
4-
5-
def pip_install_requirements():
6-
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
7-
8-
def spacy_download(language):
9-
pass
10-
11-
def preprocess():
12-
pass
1+
from utils.python_utils import pip_install_requirements
132

143
if __name__ == '__main__':
154
pip_install_requirements()
16-
spacy_download('')
17-
preprocess()

torchbenchmark/models/hf_T5_base/install.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
1-
2-
import subprocess
3-
import sys
41
import os
2+
from utils.python_utils import pip_install_requirements
53
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model
64

7-
def pip_install_requirements():
8-
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
9-
105
if __name__ == '__main__':
116
pip_install_requirements()
127
patch_transformers()

torchbenchmark/models/hf_T5_large/install.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
1-
2-
import subprocess
3-
import sys
41
import os
52
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model
6-
7-
def pip_install_requirements():
8-
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
3+
from utils.python_utils import pip_install_requirements
94

105
if __name__ == '__main__':
116
pip_install_requirements()

torchbenchmark/models/hf_Whisper/install.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,7 @@
1-
import subprocess
2-
import sys
31
import os
2+
from utils.python_utils import pip_install_requirements
43
from torchbenchmark.util.framework.huggingface.patch_hf import patch_transformers, cache_model
54

6-
def pip_install_requirements():
7-
subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', '-r', 'requirements.txt'])
8-
95
if __name__ == '__main__':
106
pip_install_requirements()
117
patch_transformers()
Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,7 @@
11
import os.path
2-
import subprocess
3-
import sys
2+
from utils.python_utils import pip_install_requirements
43

54
CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
65

7-
86
def install_pytorch_geometric():
9-
pip_install_requirements()
10-
11-
12-
def pip_install_requirements():
13-
requirements_file = os.path.join(CURRENT_DIR, "requirements.txt")
14-
subprocess.check_call(
15-
[sys.executable, "-m", "pip", "install", "-q", "-r", requirements_file]
16-
)
7+
pip_install_requirements(os.path.join(CURRENT_DIR, "requirements.txt"))

utils/python_utils.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import argparse
1+
import warnings
2+
from pathlib import Path
23
import subprocess
34

45
DEFAULT_PYTHON_VERSION = "3.11"
@@ -14,14 +15,43 @@
1415
"pytorch_url": "cp311",
1516
},
1617
}
18+
REPO_DIR = Path(__file__).parent.parent
1719

1820

1921
def create_conda_env(pyver: str, name: str):
2022
command = ["conda", "create", "-n", name, "-y", f"python={pyver}"]
2123
subprocess.check_call(command)
2224

2325

26+
def pip_install_requirements(requirements_txt="requirements.txt", continue_on_fail=False):
27+
import sys
28+
constraints_file = REPO_DIR.joinpath("build", "contraints.txt")
29+
if not constraints_file.exists():
30+
warnings.warn("contraints.txt could not be found, please rerun install.py.")
31+
constraints_parameters = []
32+
else:
33+
constraints_parameters = ["-c", str(constraints_file.resolve())]
34+
if not continue_on_fail:
35+
subprocess.check_call(
36+
[sys.executable, "-m", "pip", "install", "-r", requirements_txt] + constraints_parameters,
37+
)
38+
return True, None
39+
try:
40+
subprocess.run(
41+
[sys.executable, "-m", "pip", "install", "-r", requirements_txt] + constraints_parameters,
42+
check=True,
43+
stdout=subprocess.PIPE,
44+
stderr=subprocess.STDOUT,
45+
)
46+
except subprocess.CalledProcessError as e:
47+
return (False, e.output)
48+
except Exception as e:
49+
return (False, e)
50+
return True, None
51+
52+
2453
if __name__ == "__main__":
54+
import argparse
2555
parser = argparse.ArgumentParser()
2656
parser.add_argument(
2757
"--pyver",

0 commit comments

Comments
 (0)