Merge branch 'main' into requirements_fixed

Author: JasonMts

.github/workflows/userbenchmark-a100.yml

@@ -21,52 +21,47 @@ jobs:
       TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.TORCHBENCH_USERBENCHMARK_SCRIBE_GRAPHQL_ACCESS_TOKEN }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      SETUP_SCRIPT: "/workspace/setup_instance.sh"
     steps:
       - name: Checkout TorchBench
         uses: actions/checkout@v3
-        with:
-          path: benchmark
-      - name: Clone and setup conda env
+      - name: Install Conda
         run: |
-          CONDA_ENV=${BASE_CONDA_ENV} . "${SETUP_SCRIPT}"
-          conda create --name "${CONDA_ENV}" --clone "${BASE_CONDA_ENV}"
+          bash ./.ci/torchbench/install-conda.sh
       - name: Install TorchBench
         run: |
-          set -x
-          . "${SETUP_SCRIPT}"
-          pushd benchmark
-          python install.py
+          bash ./.ci/torchbench/install.sh
       - name: Run user benchmark
         run: |
           set -x
-          . "${SETUP_SCRIPT}"
+          . ${HOME}/miniconda3/etc/profile.d/conda.sh
+          conda activate "${CONDA_ENV}"
+
           # remove old results
           if [ -d benchmark-output ]; then rm -Rf benchmark-output; fi
-          pushd benchmark
+ 
           if [ -d .userbenchmark ]; then rm -Rf .userbenchmark; fi
           MANUAL_WORKFLOW="${{ github.event.inputs.userbenchmark_name }}"
           if [ -z "${MANUAL_WORKFLOW}" ]; then
             # Figure out what userbenchmarks we should run, and run it
             python ./.github/scripts/userbenchmark/schedule-benchmarks.py --platform ${PLATFORM_NAME}
             if [ -d ./.userbenchmark ]; then
-              cp -r ./.userbenchmark ../benchmark-output
+              cp -r ./.userbenchmark benchmark-output
             else
-              mkdir ../benchmark-output
+              mkdir benchmark-output
             fi
           else
             python run_benchmark.py "${{ github.event.inputs.userbenchmark_name }}" ${{ github.event.inputs.userbenchmark_options }}
-            cp -r ./.userbenchmark/"${{ github.event.inputs.userbenchmark_name }}" ../benchmark-output
+            cp -r ./.userbenchmark/"${{ github.event.inputs.userbenchmark_name }}" benchmark-output
+            ls -las benchmark-output
+            pwd
           fi
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
           name: TorchBench result
-          path: benchmark-output/
+          path: benchmark-output
       - name: Upload result jsons to Scribe and S3
         run: |
-          . "${SETUP_SCRIPT}"
-          pushd benchmark
           RESULTS=($(find ${PWD}/../benchmark-output -name "metrics-*.json" -maxdepth 2 | sort -r))
           echo "Uploading result jsons: ${RESULTS}"
           for r in ${RESULTS[@]}; do
@@ -76,6 +71,5 @@ jobs:
       - name: Clean up Conda env
         if: always()
         run: |
-          . "${SETUP_SCRIPT}"
-          conda deactivate && conda deactivate
+          . ${HOME}/miniconda3/etc/profile.d/conda.sh
           conda remove -n "${CONDA_ENV}" --all

userbenchmark/dynamo/dynamobench/_dynamo/testing.py

@@ -8,17 +8,8 @@
 import sys
 import types
 import unittest
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    overload,
-    Sequence,
-    TypeVar,
-    Union,
-)
+from collections.abc import Sequence
+from typing import Any, Callable, Optional, overload, TypeVar, Union
 from typing_extensions import ParamSpec
 from unittest.mock import patch
 
@@ -83,7 +74,7 @@ def extract_graph_backend(_gm, *args, **kwargs):  # type: ignore[no-untyped-def]
 
 def collect_results(
     model: torch.nn.Module, prediction: Any, loss: Any, example_inputs: Any
-) -> List[Any]:
+) -> list[Any]:
     results = []
     results.append(prediction)
     results.append(loss)
@@ -140,7 +131,7 @@ def reduce_to_scalar_loss(out: torch.Tensor) -> torch.Tensor:
 
 @overload
 def reduce_to_scalar_loss(
-    out: Union[List[Any], tuple[Any, ...], Dict[Any, Any]]
+    out: Union[list[Any], tuple[Any, ...], dict[Any, Any]]
 ) -> float:
     ...
 
@@ -186,7 +177,7 @@ def debug_insert_nops(
 ) -> Optional[GuardedCode]:
     """used to debug jump updates"""
 
-    def insert_nops(instructions: List[Any], code_options: Any) -> None:
+    def insert_nops(instructions: list[Any], code_options: Any) -> None:
         instructions.insert(0, create_instruction("NOP"))
         instructions.insert(0, create_instruction("NOP"))
 
@@ -222,7 +213,7 @@ def __init__(self) -> None:
         self.op_count = 0
 
     def __call__(
-        self, gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+        self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
     ) -> Callable[..., Any]:
         self.frame_count += 1
         for node in gm.graph.nodes:
@@ -240,10 +231,10 @@ def __init__(self, backend: str) -> None:
         self.frame_count = 0
         self.op_count = 0
         self.backend = backend
-        self.graphs: List[torch.fx.GraphModule] = []
+        self.graphs: list[torch.fx.GraphModule] = []
 
     def __call__(
-        self, gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+        self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
     ) -> Callable[..., Any]:
         from .backends.registry import lookup_backend
 
@@ -264,34 +255,34 @@ def clear(self) -> None:
 # we can assert on
 class EagerAndRecordGraphs:
     def __init__(self) -> None:
-        self.graphs: List[torch.fx.GraphModule] = []
+        self.graphs: list[torch.fx.GraphModule] = []
 
     def __call__(
-        self, gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+        self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
     ) -> Callable[..., Any]:
         self.graphs.append(gm)
         return gm.forward
 
 
 class AotEagerAndRecordGraphs:
     def __init__(self) -> None:
-        self.graphs: List[torch.fx.GraphModule] = []
-        self.fw_graphs: List[torch.fx.GraphModule] = []
-        self.bw_graphs: List[torch.fx.GraphModule] = []
+        self.graphs: list[torch.fx.GraphModule] = []
+        self.fw_graphs: list[torch.fx.GraphModule] = []
+        self.bw_graphs: list[torch.fx.GraphModule] = []
 
     def __call__(
-        self, gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+        self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
     ) -> Callable[..., Any]:
         self.graphs.append(gm)
 
         def fw_compiler(
-            gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+            gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
         ) -> Callable[..., Any]:
             self.fw_graphs.append(gm)
             return gm.forward
 
         def bw_compiler(
-            gm: torch.fx.GraphModule, example_inputs: List[torch.Tensor]
+            gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor]
         ) -> Callable[..., Any]:
             self.bw_graphs.append(gm)
             return gm.forward
@@ -360,7 +351,7 @@ def standard_test(
 
 
 def dummy_fx_compile(
-    gm: fx.GraphModule, example_inputs: List[torch.Tensor]
+    gm: fx.GraphModule, example_inputs: list[torch.Tensor]
 ) -> Callable[..., Any]:
     return gm.forward