Suppress csv creation on cold-start phase of --warm-start-latency (#125953)

Summary:
It seems that most (all?) of our utilities for examining benchmark output expect single-line entries per benchmark. The way the --warm-start-latency flag is currently implemented, it means that we'll see two entries for every benchmark run (one for the warm-up run and one for the actual run). This PR adds a --disable-output flag that we can use for the first run to suppress populating the csv. This way, the existing utilities like `benchmarks/dynamo/check_accuracy.py` will function without any changes.

X-link: pytorch/pytorch#125953
Approved by: https://github.com/desertfire
ghstack dependencies: #125917

Reviewed By: huydhn

Differential Revision: D57421488

Pulled By: masnesral

fbshipit-source-id: 02c6e7500fbddcb7eb474e2170c3a3ae5f369487

Author: masnesral

userbenchmark/dynamo/dynamobench/common.py

@@ -108,6 +108,7 @@
 current_onnx_compiler = ""
 current_batch_size = None
 output_filename = None
+disable_output = False
 
 MAX_DOWNLOAD_ATTEMPTS = 5
 
@@ -306,6 +307,9 @@ def load_model_from_path(path_and_class_str):
 
 
 def output_csv(filename, headers, row):
+    global disable_output
+    if disable_output:
+        return
     if os.path.exists(filename):
         with open(filename) as fd:
             lines = list(csv.reader(fd)) or [[]]
@@ -3212,6 +3216,11 @@ def get_example_inputs(self):
         "--output-directory",
         help="Overrides the directory to place output files.",
     )
+    parser.add_argument(
+        "--disable-output",
+        action="store_true",
+        help="Disable writing of output files, e.g., for warm-up runs",
+    )
     parser.add_argument(
         "--baseline",
         help="Compare with a prior --output",
@@ -3612,7 +3621,8 @@ def main(runner, original_dir=None, args=None):
             cmd.remove("--warm-start-latency")
 
             print(f"Performing cold-start run for {args.only}")
-            subprocess.check_call(cmd + ["--repeat=1"], timeout=args.timeout, env=env)
+            warmup_cmd = cmd + ["--repeat=1", "--disable-output"]
+            subprocess.check_call(warmup_cmd, timeout=args.timeout, env=env)
 
             print(f"Performing warm-start run for {args.only}")
             subprocess.check_call(cmd, timeout=args.timeout, env=env)
@@ -3821,9 +3831,12 @@ def run(runner, args, original_dir=None):
         runner.skip_models.clear()
 
     experiment = null_experiment
-    global current_name, current_device, current_batch_size, output_filename, optimize_ctx, current_onnx_compiler
+    global current_name, current_device, current_batch_size, output_filename, disable_output, optimize_ctx, current_onnx_compiler
     optimize_ctx = contextlib.nullcontext()
 
+    if args.disable_output:
+        disable_output = True
+
     if args.overhead:
         optimize_ctx = torch._dynamo.optimize(dummy_fx_compile, nopython=args.nopython)
         experiment = speedup_experiment