do not cast all inputs in benchmarks (pytorch#108456)

Fixes why stable diffusion is not showing up in inference dashboard even though it shows up in training dashboard The reason is stable diffusion in torchbench has a line like `input_tensor = input_tensor.long().to(self.device)` and if you cast this to a bfloat16 the inference will fail <img width="1705" alt="Screenshot 2023-09-01 at 4 37 49 PM" src="https://github.com/pytorch/pytorch/assets/3282513/ada0d381-1af0-4378-8e8b-2375b39c3713"> Pull Request resolved: pytorch#108456 Approved by: https://github.com/cpuhrsch

Author: msaroufim

benchmarks/dynamo/common.py

@@ -305,6 +305,8 @@ class CI(NamedTuple):
  "dlrm",
 }
 
+DO_NOT_CAST_INPUTS = {"stable_diffusion"}
+
 
 def model_specified_by_path(path_and_class_str):
  return ":" in path_and_class_str
@@ -3482,8 +3484,11 @@ def detect_and_mark_batch(t):
  torch.cuda.set_per_process_memory_fraction(
  args.per_process_memory_fraction
  )
+ if model_name in DO_NOT_CAST_INPUTS:
+ model, _ = runner.cast_based_on_args(model, example_inputs)
 
- model, example_inputs = runner.cast_based_on_args(model, example_inputs)
+ else:
+ model, example_inputs = runner.cast_based_on_args(model, example_inputs)
  runner.run_one_model(
  name,
  model,