Hardcode the numpy version (#2316)

Summary:
When there is a big numpy version bump (1.21.2 -> 2.0.0), `pip install` will somehow automatically upgrade numpy version to 2.0.0 even when the old version (1.21.2) has been installed.

Therefore, we have to hardcode numpy version both globally and for the models whose install will cause numpy to unexpectedly upgrade. Since downstream CI supports Python 3.8 as the lowest supported Python version, we have to pin numpy version to the lowest version used in the downstream CI.

Pull Request resolved: #2316

Reviewed By: aaronenyeshi

Differential Revision: D58787422

Pulled By: xuzhao9

fbshipit-source-id: 1fdc667478350d831bdd83d434c3e4c5620b7e71

Author: xuzhao9

requirements.txt

@@ -15,7 +15,10 @@ transformers==4.38.1
 MonkeyType
 psutil
 pyyaml
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 opencv-python
 submitit
 pynvml

torchbenchmark/models/Background_Matting/requirements.txt

@@ -1,4 +1,7 @@
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 opencv-python
 pandas
 Pillow

torchbenchmark/models/hf_Whisper/requirements.txt

@@ -1 +1,5 @@
-numba
+numba
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'

torchbenchmark/models/tacotron2/requirements.txt

@@ -1,4 +1,7 @@
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 inflect
 scipy
 Unidecode

torchbenchmark/util/framework/detectron2/requirements.txt

@@ -1,3 +1,6 @@
 git+https://github.com/facebookresearch/detectron2.git@0df2d73d0013db7de629602c23cc120219b4f2b8
 omegaconf==2.3.0
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'

utils/build_requirements.txt

@@ -0,0 +1,6 @@
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
+psutil
+pyyaml

utils/cuda_utils.py

@@ -9,6 +9,7 @@
 
 # defines the default CUDA version to compile against
 DEFAULT_CUDA_VERSION = "12.4"
+REPO_ROOT = Path(__file__).parent.parent
 
 CUDA_VERSION_MAP = {
     "12.4": {
@@ -17,11 +18,9 @@
     },
 }
 PIN_CMAKE_VERSION = "3.22.*"
-# the numpy version needs to be consistent with
-# https://github.com/pytorch/builder/blob/e66e48f9b1968213c6a7ce3ca8df6621435f0a9c/wheel/build_wheel.sh#L146
-PIN_NUMPY_VERSION = "1.23.5"
-TORCHBENCH_TORCH_NIGHTLY_PACKAGES = ["torch", "torchvision", "torchaudio"]
 
+TORCHBENCH_TORCH_NIGHTLY_PACKAGES = ["torch", "torchvision", "torchaudio"]
+BUILD_REQUIREMENTS_FILE = REPO_ROOT.joinpath("utils", "build_requirements.txt")
 
 def _nvcc_output_match(nvcc_output, target_cuda_version):
     regex = "release (.*),"
@@ -153,9 +152,8 @@ def install_torch_build_deps(cuda_version: str):
     build_deps = ["ffmpeg"]
     cmd = ["conda", "install", "-y"] + build_deps
     subprocess.check_call(cmd)
-    # pip deps
-    pip_deps = [f"numpy=={PIN_NUMPY_VERSION}"]
-    cmd = ["pip", "install"] + pip_deps
+    # pip build deps
+    cmd = ["pip", "install", "-r"] + str(BUILD_REQUIREMENTS_FILE.resolve())
     subprocess.check_call(cmd)
     # conda forge deps
     # ubuntu 22.04 comes with libstdcxx6 12.3.0